Eval runs

Resolve or create a file-owned eval suite

Resolve a file-owned suite by declared id within the project, or create one. Lookup is by (projectId, declaredSuiteId) and never by name. A UI-authored suite has no declared id and cannot be claimed. The inspector parses the suite file; this body is the declared identity, source hash, optional provenance, and hosted settings — not the raw file.

201 on first create; 200 on a later update of the same declared id.

POST

projects

{projectId}

eval-suites

from-file

Resolve or create a file-owned eval suite

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "declaredSuiteId": "<string>",
  "name": "<string>",
  "sourceHash": "<string>",
  "description": "<string>",
  "environment": {
    "servers": [
      "<string>"
    ],
    "serverBindings": [
      {
        "serverName": "<string>",
        "projectServerId": "<string>"
      }
    ]
  },
  "verdictPolicyVersion": 2,
  "minIterations": 5,
  "defaultPassCriteria": {
    "minimumPassRatePercent": 50,
    "minimumPassRate": 50
  }
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file"

payload = {
    "declaredSuiteId": "<string>",
    "name": "<string>",
    "sourceHash": "<string>",
    "description": "<string>",
    "environment": {
        "servers": ["<string>"],
        "serverBindings": [
            {
                "serverName": "<string>",
                "projectServerId": "<string>"
            }
        ]
    },
    "verdictPolicyVersion": 2,
    "minIterations": 5,
    "defaultPassCriteria": {
        "minimumPassRatePercent": 50,
        "minimumPassRate": 50
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    declaredSuiteId: '<string>',
    name: '<string>',
    sourceHash: '<string>',
    description: '<string>',
    environment: {
      servers: ['<string>'],
      serverBindings: [{serverName: '<string>', projectServerId: '<string>'}]
    },
    verdictPolicyVersion: 2,
    minIterations: 5,
    defaultPassCriteria: {minimumPassRatePercent: 50, minimumPassRate: 50}
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'declaredSuiteId' => '<string>',
    'name' => '<string>',
    'sourceHash' => '<string>',
    'description' => '<string>',
    'environment' => [
        'servers' => [
                '<string>'
        ],
        'serverBindings' => [
                [
                                'serverName' => '<string>',
                                'projectServerId' => '<string>'
                ]
        ]
    ],
    'verdictPolicyVersion' => 2,
    'minIterations' => 5,
    'defaultPassCriteria' => [
        'minimumPassRatePercent' => 50,
        'minimumPassRate' => 50
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file"

	payload := strings.NewReader("{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "created": true,
  "suite": {
    "id": "<string>",
    "environment": {
      "servers": [
        "<string>"
      ],
      "computerEnvironment": {
        "id": "<string>",
        "name": "<string>"
      }
    },
    "hosts": [
      {
        "id": "<string>",
        "name": "<string>",
        "servers": [
          "<string>"
        ]
      }
    ],
    "settings": {
      "minimumAccuracy": 123,
      "matchOptions": {},
      "checks": [
        {}
      ],
      "judge": {
        "enabled": true,
        "model": "<string>",
        "autoRun": true,
        "threshold": 123,
        "severity": "warn",
        "groundedness": {
          "role": "advisory",
          "model": "<string>",
          "threshold": 123,
          "severity": "warn"
        },
        "rubric": {
          "criteria": [
            {
              "id": "<string>",
              "label": "<string>",
              "description": "<string>",
              "required": true
            }
          ]
        }
      },
      "minimumIterations": 123,
      "verdictPolicyVersion": 2,
      "verdictPolicyDefaults": {
        "repetitions": 2,
        "passThreshold": 0.5,
        "validity": {
          "minEligibleTrials": 2,
          "minCompletionRate": 0.5,
          "maxEvaluatorErrorRate": 0.5
        }
      },
      "policy": "legacy",
      "qualityGate": {
        "baseline": {
          "kind": "run",
          "runId": "<string>"
        },
        "maximumPassRateDrop": 123,
        "noDeterministicRegressions": true,
        "maximumP95LatencyIncreaseMs": 123,
        "noGatingScoreErrors": true
      }
    },
    "schedule": {
      "enabled": true,
      "intervalMinutes": 5042,
      "environmentId": "<string>",
      "state": "active",
      "createdBy": "<string>",
      "nextDueAt": 123,
      "consecutiveFailures": 123
    },
    "declaredId": "<string>",
    "managedBy": "ci",
    "name": "<string>",
    "description": "<string>",
    "projectId": "<string>",
    "executionConfig": {
      "model": "<string>",
      "systemPrompt": "<string>",
      "temperature": 123
    },
    "environmentIds": [
      "<string>"
    ],
    "revisionNumber": 123,
    "createdAt": 123,
    "updatedAt": 123
  }
}

Authorizations

Authorization

string

header

required

MCPJam API key (sk_…). Create one at Settings → API keys. Guest sessions cannot use the API, and API keys cannot manage other API keys.

Headers

x-mcpjam-eval-vocabulary

enum<string>

Which vocabulary this request and its response speak. Absent means 1, which is byte-for-byte today's contract: the same request fields, the same refusals, the same response projection. 2 is the canonical vocabulary. Any other value is a 400 with code: "VALIDATION_ERROR".

Today it decides one thing: the spelling of an evaluator's policy role. Vocabulary 1 accepts and returns gating; vocabulary 2 accepts both spellings and returns the canonical required. Sending required without the header is a 400, deliberately — vocabulary 1 is not widened to meet vocabulary 2 half way, because a boundary that accepts a spelling it does not announce is one two implementations can disagree about.

A response that varies by vocabulary sends Vary: x-mcpjam-eval-vocabulary.

Available options:

1,

2

Path Parameters

projectId

string

required

ID of the hosted project that contains the server.

Body

application/json

Declared identity and hosted settings for a file-owned suite. The inspector parses the suite file; this is not the raw file.

declaredSuiteId

string

required

The file's suite.id. Lookup is by this id within the project, never by name.

name

string

required

sourceHash

string

required

SHA-256 hex of the suite-file bytes. Lowercase, 64 characters.

Pattern: ^[a-f0-9]{64}$

description

string

provenance

object

Show child attributes

environment

object

Show child attributes

defaultConfig

object

Show child attributes

verdictPolicyVersion

any

Verdict policy version used by the suite file. Must be supplied with verdictPolicyDefaults.

verdictPolicyDefaults

object

Show child attributes

minIterations

integer

Required range: 1 <= x <= 10

defaultPassCriteria

object

The floor a run must clear, as a PERCENT in [0, 100] — 80 means 80%. minimumPassRatePercent is the canonical spelling (the unit is in the name); minimumPassRate is the deprecated alias for it. Send exactly one. THE NAME DISAMBIGUATES THE UNIT: a value in (0, 1) is accepted on minimumPassRatePercent, where it unambiguously means a sub-1% floor the evaluator can act on, and REJECTED on the bare minimumPassRate, where 0.8 cannot be told apart from a fraction sent by mistake and would silently make the gate unfailable. Neither field ever reinterprets a value. A per-case passThreshold IS a fraction; this suite/run floor is not.

Option 1
Option 2

Show child attributes

Response

The existing file-owned suite was updated.

created

boolean

required

True on the first upload of this declared id in the project; false on a later update of the same suite.

suite

object

required

One eval suite's full configuration. Distinct from the EvalSuite summary returned by the list route, which carries run rollups instead of settings.

Show child attributes

⌘I

Resolve or create a file-owned eval suite

curl --request POST \
  --url https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "declaredSuiteId": "<string>",
  "name": "<string>",
  "sourceHash": "<string>",
  "description": "<string>",
  "environment": {
    "servers": [
      "<string>"
    ],
    "serverBindings": [
      {
        "serverName": "<string>",
        "projectServerId": "<string>"
      }
    ]
  },
  "verdictPolicyVersion": 2,
  "minIterations": 5,
  "defaultPassCriteria": {
    "minimumPassRatePercent": 50,
    "minimumPassRate": 50
  }
}
'

import requests

url = "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file"

payload = {
    "declaredSuiteId": "<string>",
    "name": "<string>",
    "sourceHash": "<string>",
    "description": "<string>",
    "environment": {
        "servers": ["<string>"],
        "serverBindings": [
            {
                "serverName": "<string>",
                "projectServerId": "<string>"
            }
        ]
    },
    "verdictPolicyVersion": 2,
    "minIterations": 5,
    "defaultPassCriteria": {
        "minimumPassRatePercent": 50,
        "minimumPassRate": 50
    }
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    declaredSuiteId: '<string>',
    name: '<string>',
    sourceHash: '<string>',
    description: '<string>',
    environment: {
      servers: ['<string>'],
      serverBindings: [{serverName: '<string>', projectServerId: '<string>'}]
    },
    verdictPolicyVersion: 2,
    minIterations: 5,
    defaultPassCriteria: {minimumPassRatePercent: 50, minimumPassRate: 50}
  })
};

fetch('https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'declaredSuiteId' => '<string>',
    'name' => '<string>',
    'sourceHash' => '<string>',
    'description' => '<string>',
    'environment' => [
        'servers' => [
                '<string>'
        ],
        'serverBindings' => [
                [
                                'serverName' => '<string>',
                                'projectServerId' => '<string>'
                ]
        ]
    ],
    'verdictPolicyVersion' => 2,
    'minIterations' => 5,
    'defaultPassCriteria' => [
        'minimumPassRatePercent' => 50,
        'minimumPassRate' => 50
    ]
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file"

	payload := strings.NewReader("{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://app.mcpjam.com/api/v1/projects/{projectId}/eval-suites/from-file")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"declaredSuiteId\": \"<string>\",\n  \"name\": \"<string>\",\n  \"sourceHash\": \"<string>\",\n  \"description\": \"<string>\",\n  \"environment\": {\n    \"servers\": [\n      \"<string>\"\n    ],\n    \"serverBindings\": [\n      {\n        \"serverName\": \"<string>\",\n        \"projectServerId\": \"<string>\"\n      }\n    ]\n  },\n  \"verdictPolicyVersion\": 2,\n  \"minIterations\": 5,\n  \"defaultPassCriteria\": {\n    \"minimumPassRatePercent\": 50,\n    \"minimumPassRate\": 50\n  }\n}"

response = http.request(request)
puts response.read_body

{
  "created": true,
  "suite": {
    "id": "<string>",
    "environment": {
      "servers": [
        "<string>"
      ],
      "computerEnvironment": {
        "id": "<string>",
        "name": "<string>"
      }
    },
    "hosts": [
      {
        "id": "<string>",
        "name": "<string>",
        "servers": [
          "<string>"
        ]
      }
    ],
    "settings": {
      "minimumAccuracy": 123,
      "matchOptions": {},
      "checks": [
        {}
      ],
      "judge": {
        "enabled": true,
        "model": "<string>",
        "autoRun": true,
        "threshold": 123,
        "severity": "warn",
        "groundedness": {
          "role": "advisory",
          "model": "<string>",
          "threshold": 123,
          "severity": "warn"
        },
        "rubric": {
          "criteria": [
            {
              "id": "<string>",
              "label": "<string>",
              "description": "<string>",
              "required": true
            }
          ]
        }
      },
      "minimumIterations": 123,
      "verdictPolicyVersion": 2,
      "verdictPolicyDefaults": {
        "repetitions": 2,
        "passThreshold": 0.5,
        "validity": {
          "minEligibleTrials": 2,
          "minCompletionRate": 0.5,
          "maxEvaluatorErrorRate": 0.5
        }
      },
      "policy": "legacy",
      "qualityGate": {
        "baseline": {
          "kind": "run",
          "runId": "<string>"
        },
        "maximumPassRateDrop": 123,
        "noDeterministicRegressions": true,
        "maximumP95LatencyIncreaseMs": 123,
        "noGatingScoreErrors": true
      }
    },
    "schedule": {
      "enabled": true,
      "intervalMinutes": 5042,
      "environmentId": "<string>",
      "state": "active",
      "createdBy": "<string>",
      "nextDueAt": 123,
      "consecutiveFailures": 123
    },
    "declaredId": "<string>",
    "managedBy": "ci",
    "name": "<string>",
    "description": "<string>",
    "projectId": "<string>",
    "executionConfig": {
      "model": "<string>",
      "systemPrompt": "<string>",
      "temperature": 123
    },
    "environmentIds": [
      "<string>"
    ],
    "revisionNumber": 123,
    "createdAt": 123,
    "updatedAt": 123
  }
}

API Guides

Endpoints

Resolve or create a file-owned eval suite

Authorizations

Headers

Path Parameters

Body

Response