"""Run the resyn task suite on a model and post its answers to resyn.cloud. Made for agents and people alike. Standard library only, so it runs anywhere Python 3.9+ does: python3 resyn-bench.py ~/models/Qwen3.5-4B-Q8_K_XL.gguf --agent "Claude Code" # a local file, via llama-server python3 resyn-bench.py --provider openrouter --model openai/gpt-4o-mini # any model behind an API python3 resyn-bench.py --provider ollama --model qwen3:8b --size medium # anything already served locally python3 resyn-bench.py --dry-run ... # ask the model, show what would be posted, post nothing python3 resyn-bench.py --mcp # serve the same as MCP tools on stdin/stdout It fetches the prompts of one size of the suite (--size fast, medium or full; fast when none is named), asks the model each one, and posts the answers with the tokens and time each took, as that size. The site grades the answers itself, so nobody, person or model, types a score. Every task of the size is answered: the scored tasks in the suite's order, then any the suite does not score. It records itself as the harness and asks the server what runtime it is. Set RESYN_TOKEN to your account's API token (from your profile page) to post under your name; without it the run is anonymous. """ from __future__ import annotations import argparse import hashlib import http.client import ipaddress import json import os import platform import re import shutil import socket import subprocess import sys import time import urllib.error import urllib.parse import urllib.request from pathlib import Path SITE = "https://resyn.cloud" USER_AGENT = "resyn-bench/1" HARNESS = "resyn-bench 2" RETRY_STATUS = (429, 500, 502, 503, 504) # Worth asking again; any other 4xx will not change. # The app refusing a tools request answers with its own words, under one of these statuses: 400 or 422 for the request, or 404 or 405 # for the address or the method (OpenRouter answers 404 when no endpoint supports tools). Every other status is an error for the task: # a 5xx is the server failing, which says nothing about the tools, and 401, 402, 403, 408, 409, 413 and 429 are about who is asking, # the credit, a conflict, a timeout, a request size or a rate limit. REFUSAL_STATUSES = (400, 404, 405, 422) # The app's words for a refusal, in any case. Only these forms make one, and each says the app does not support the tools. "Tools" here # also stands for tool use, tool calls, tool calling, function calls and function calling. A form may have other words before or after # it. Nothing else does: a size, model, key, account, missing-model, markup or unknown message is an error for the task, and so is a # message that names one tool or one option (_NAMED). llms.txt step 3 and docs/operations.md state these forms in plain words; # tests/test_agents.py holds the copy to this list, form by form and near miss by near miss. _TOOLS = r"(?:tools|tool\s+(?:requests?|use|calls?|calling)|function\s+(?:calls?|calling))" _NOT_SUPPORTED = r"(?:not\s+supported|unsupported|not\s+permitted)" _DOES_NOT_SUPPORT = r"(?:does|do|did)\s+not\s+support|(?:doesn|don)['\u2019]t\s+support" _NAME = r"[A-Za-z0-9.:/-]+" # A model, server or endpoint's name in a refusal: no underscore, and a pair of quotes or brackets may go round it. REFUSAL_WORDS = re.compile( rf"\b(?:{_DOES_NOT_SUPPORT})\s+(?:the\s+)?{_TOOLS}\b" # does not support tools; doesn't support the tools # tools are not supported; tools are, for this model, not supported: 1 to 80 characters between the commas, the last not a space rf"|\b{_TOOLS}\s+(?:(?:is|are|was|were)\s+|(?:is|are),[^,.;:]{{1,80}}(?['\"`])tools(?P=opening)|tools\b)" rf"|\bunsupported\s+(?:tool\s+(?:requests?|use|calling)|function\s+calling)\b" # unsupported tool request: the generic words only rf"|\b(?:model|server|endpoint)\s+(?:'{_NAME}'|\"{_NAME}\"|`{_NAME}`|<{_NAME}>|{_NAME})\s+(?:is\s+|are\s+)?{_NOT_SUPPORTED}\s+(?:for|with)\s+(?:the\s+)?{_TOOLS}\b" # model qwen3 is not supported for tools rf"|\binput\s+token\s+counting\s+(?:is\s+|are\s+)?{_NOT_SUPPORTED}\s+(?:for|with)\s+(?:the\s+)?{_TOOLS}\b" # input token counting is not supported for tools rf"|(?P['\"`])tools(?P=quote)\s+(?:is\s+|are\s+)?not\s+permitted\b" # 'tools' is not permitted rf"|\btools\s+(?:(?:param|parameter|argument)\s+)?requires?\s+(?:the\s+)?--jinja\b" # tools param requires --jinja rf"|\bno\s+\w+(?:\s+\w+)?\s+(?:that\s+)?supports?\s+(?:the\s+)?{_TOOLS}\b", # no endpoints found that support tool use re.IGNORECASE, ) # A name of one tool or one option makes a message an error even in a form: a word with an underscore (tool_choice, web_search), one word # in parentheses, square or angle brackets or in quotes ((stream), 'required'), or the word after tool, option or options (tool choice, # option stream). The tools do not name one: tool use, tool request(s), tool call(s) and tool calling, and tools after option. A model, # server or endpoint's name wrapped in quotes or angle brackets, and tools wrapped in quotes, are taken out first, so they are no names. _WRAPPED_MODEL = re.compile(r"""\b(?:model|server|endpoint)\s+(?:'[^'\n]*'|"[^"\n]*"|`[^`\n]*`|<[^<>\n]*>)(?=\s)""", re.IGNORECASE) _WRAPPED_TOOLS = re.compile(r"""(['"`])tools\1""", re.IGNORECASE) _NAMED = re.compile( r"\w*_\w*" # tool_choice, web_search r"|\([^\s()]+\)|\[[^\s\[\]]+\]|<[^\s<>]+>" # (stream), [stream], r"|(? bool: """Does the message name one tool or one option? See _NAMED.""" return bool(_NAMED.search(_WRAPPED_TOOLS.sub("tools", _WRAPPED_MODEL.sub("model", message)))) # A size or a limit is an error for the task even where the same message says the tools are not supported: the app refused the request # for its size. A figure in tokens counts; the word alone does not, since input token counting is a feature and not a size. _SIZED = re.compile(r"(?i)\b(?:context\w*|exceed\w*|overflow\w*|maximum|max|limit\w*|too\s+(?:large|long|big|many)|above|beyond|length)\b" r"|\b\d[\d,]*\s*-?\s*tokens?\b") # Every HTML element's name. A tag of one of these, its name followed by a space, a slash or the closing bracket, is markup. _HTML_ELEMENTS = ( "a abbr acronym address applet area article aside audio b base basefont bdi bdo bgsound big blink blockquote body br button " "canvas caption center cite code col colgroup data datalist dd del details dfn dialog dir div dl dt em embed fieldset figcaption " "figure font footer form frame frameset h1 h2 h3 h4 h5 h6 head header hgroup hr html i iframe image img input ins isindex kbd " "keygen label legend li link listing main map mark marquee math menu menuitem meta meter multicol nav nextid nobr noembed noframes noscript " "object ol optgroup option output p param picture plaintext pre progress q rb rbc rp rt rtc ruby s samp script search section " "select selectedcontent slot small source spacer span strike strong style sub summary sup svg table tbody td template textarea tfoot th thead " "time title tr track tt u ul var video wbr xmp" ).split() # Markup from a proxy, a gateway or a web server: a page, a fragment such as nginx's error block, or a tag inside a message. Its words are # not the app's, so it is never a refusal, whatever it says. A model's name in angle brackets is the app's words, even when the name is an # element's (): _has_markup takes such a name out first. is no tag, because the name goes on past the element's name. _MARKUP = re.compile(r"(?i)<[!?]|]|$)") MAX_RETRIES, MAX_WAIT = 3, 60 _OLLAMA_BLOB = re.compile(r"^sha256[-:]([0-9a-f]{64})$") CONTEXTS = (32768, 16384, 8192) # llama-server's context when the caller sets none: the largest that fits. _COMMON_GB = (4, 8, 12, 16, 18, 24, 32, 36, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024) class BenchError(Exception): """A problem worth telling the caller about in plain words.""" class ServerDied(BenchError): """llama-server exited while loading: usually the model and its context did not fit in memory.""" class Unreachable(BenchError): """The model's endpoint could not be connected to at all: asking it 33 more times would not help.""" class Refused(BenchError): """The app answered a request that carried tools with an error of its own, so it will not run them. The reason is its words.""" def __init__(self, reason: str): super().__init__(reason) self.reason = reason def _wait_seconds(attempt: int, retry_after: str | None = None) -> float: """How long to wait before retry number `attempt` (1-based): the server's Retry-After, else 2, 4, 8 s; at most MAX_WAIT.""" try: wait = float(retry_after) if retry_after else 2.0 * 2 ** (attempt - 1) except ValueError: wait = 2.0 * 2 ** (attempt - 1) return max(0.0, min(wait, MAX_WAIT)) # --- Talking to the site --------------------------------------------------------------------------------------- def _credential(value: str, name: str) -> str: """A key or token as it goes in an Authorization header. The newline a copied key often ends with is dropped. A line break inside one cannot be sent (http.client refuses the header and quotes it in its error), so it is refused here, in words that never repeat the value.""" cleaned = value.strip() if "\r" in cleaned or "\n" in cleaned: raise BenchError(f"{name} has a line break in it. Copy it again as one line") return cleaned def _parse_json(text: str): """json.loads for a reply. A reply nested too deep for the parser is not JSON, so it costs its answer or page, not the run.""" try: return json.loads(text) except RecursionError: raise ValueError("nested too deeply") from None def _request(url: str, data: bytes | None = None, token: str | None = None, timeout: float = 60) -> tuple[int, dict]: token = _credential(token, "RESYN_TOKEN") if token else None headers = {"User-Agent": USER_AGENT} if token: if not key_host_allowed(url, None): # Refused before any request, as a model's key is: plain http is for this computer alone. raise BenchError(_token_refusal(url)) headers["Authorization"] = f"Bearer {token}" if data is not None: headers["Content-Type"] = "application/json" request = urllib.request.Request(url, data=data, headers=headers, method="POST" if data is not None else "GET") send = _sender(url, None, keyed=bool(token)) # With the token: no redirect that moves it, and no proxy to this computer. try: with send(request, timeout=timeout) as response: status, text = response.status, response.read().decode("utf-8", "replace") except urllib.error.HTTPError as exc: try: return exc.code, _parse_json(exc.read().decode("utf-8") or "{}") except ValueError: return exc.code, {"error": f"HTTP {exc.code}"} # A half-sent or garbled reply is as unreachable as a closed port. A ValueError is a header or URL that http.client will not # send, and its text quotes the header, token and all. except (OSError, http.client.HTTPException, ValueError) as exc: raise BenchError(f"Could not reach {url}: {_redact(str(exc), token)}") from None try: return status, _parse_json(text or "{}") except ValueError: # A web page at that address, not an API. raise BenchError(f"{url} answered with something that is not JSON (HTTP {status}); check the address") from None # --- The file and the machine ---------------------------------------------------------------------------------- def sha256_of(path: Path) -> str: match = _OLLAMA_BLOB.match(path.name) if match: # Ollama names model blobs by their SHA-256. return match.group(1) digest = hashlib.sha256() with open(path, "rb") as stream: for chunk in iter(lambda: stream.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def _run(command: list[str], timeout: float = 10) -> str: try: return subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=True).stdout except (OSError, subprocess.SubprocessError): return "" def _marketed(gib: float) -> int: """Installed memory as sold: 62.6 GiB usable on a 64 GB machine reads as 64.""" for size in _COMMON_GB: if size * 0.85 <= gib <= size * 1.02: return size return round(gib) def system_ram_gb() -> int | None: try: if sys.platform.startswith("linux"): for line in Path("/proc/meminfo").read_text().splitlines(): if line.startswith("MemTotal:"): return _marketed(int(line.split()[1]) / 2**20) if sys.platform == "darwin": return _marketed(int(_run(["sysctl", "-n", "hw.memsize"]).strip()) / 2**30) if sys.platform == "win32": import ctypes class Status(ctypes.Structure): _fields_ = [("length", ctypes.c_ulong), ("load", ctypes.c_ulong), ("total", ctypes.c_ulonglong), ("avail", ctypes.c_ulonglong), ("total_page", ctypes.c_ulonglong), ("avail_page", ctypes.c_ulonglong), ("total_virtual", ctypes.c_ulonglong), ("avail_virtual", ctypes.c_ulonglong), ("avail_extended", ctypes.c_ulonglong)] status = Status(); status.length = ctypes.sizeof(Status) if ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): return _marketed(status.total / 2**30) except (OSError, ValueError, IndexError): pass return None def _cpu_model() -> str | None: """The processor's marketing name, e.g. 'AMD Ryzen 5 5600X 6-Core Processor'.""" try: if sys.platform.startswith("linux"): for line in Path("/proc/cpuinfo").read_text().splitlines(): if line.lower().startswith("model name"): return line.partition(":")[2].strip() or None if sys.platform == "darwin": return _run(["sysctl", "-n", "machdep.cpu.brand_string"]).strip() or None except OSError: return None return platform.processor().strip() or None def detect_hardware() -> dict: """kind, gpu and memory in GB, as far as this machine says.""" ram = system_ram_gb() if sys.platform == "darwin" and platform.machine() == "arm64": return {"kind": "apple", "chip": _run(["sysctl", "-n", "machdep.cpu.brand_string"]).strip() or None, "memory": ram} if shutil.which("nvidia-smi"): first = _run(["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"]).strip().splitlines() if first: name, _, mib = first[0].rpartition(",") try: return {"kind": "gpu", "gpu": name.strip(), "vram": _marketed(float(mib) / 1024), "ram": ram} except ValueError: pass return {"kind": "cpu", "cpu": _cpu_model(), "ram": ram} # --- The task suite: any model, through any OpenAI-compatible endpoint ---------------------------------------------- # The site's current suite (resyn/tasksuite.py, NAME). This file is served alone as resyn-bench.py, so it cannot import # that name; tests/test_suite6_b1.py holds the two equal. TASK_SUITE = "resyn-tasks@6" # The suite's sizes, smallest first (resyn/sizes.py ORDER; tests/test_suite6_bench.py holds the two equal). A run answers the # tasks of one size, and its fetch, its variant and its post all name that size. SIZES = ("fast", "medium", "full") SIZE_WORDS = "fast, medium or full" # The same names in words, for a message. # The size a run answers when the command line or the MCP tool names none: the smallest, so a run that names no size is the quick one. DEFAULT_SIZE = "fast" # A sized variant token: the seed, the size, then 24 hex digits (resyn/sizes.py TOKEN). A post names the size its variant was issued for. _SIZED_VARIANT = re.compile(rf"\d{{1,10}}\.({'|'.join(SIZES)})\.[0-9a-f]{{24}}") ENDPOINTS = { "openai": ("https://api.openai.com/v1", "OPENAI_API_KEY"), "openrouter": ("https://openrouter.ai/api/v1", "OPENROUTER_API_KEY"), "ollama": ("http://127.0.0.1:11434/v1", None), "lmstudio": ("http://127.0.0.1:1234/v1", None), "llamacpp": ("http://127.0.0.1:8080/v1", None), } HOSTED = {"api.openai.com": "OpenAI API", "openrouter.ai": "OpenRouter", "api.anthropic.com": "Anthropic API", "generativelanguage.googleapis.com": "Gemini API", "api.mistral.ai": "Mistral API", "api.together.xyz": "Together", "api.together.ai": "Together", "api.groq.com": "Groq"} # The site's provider key for each known host, so a run against --endpoint https://api.groq.com/openai/v1 is filed as Groq. HOST_PROVIDERS = {"api.openai.com": "openai", "openrouter.ai": "openrouter", "api.anthropic.com": "anthropic", "generativelanguage.googleapis.com": "google", "api.mistral.ai": "mistral", "api.together.xyz": "together", "api.together.ai": "together", "api.groq.com": "groq"} def _host(base: str) -> str: return (urllib.parse.urlsplit(base).hostname or "").casefold() def _known_host(table: dict, base: str): host = _host(base) return next((value for known, value in table.items() if host == known or host.endswith("." + known)), None) def is_loopback(base: str) -> bool: """Is this endpoint served by the machine running the script?""" host = _host(base) if host == "localhost": return True try: return ipaddress.ip_address(host).is_loopback except ValueError: return False def _listed_key_hosts() -> frozenset[str]: """The host names in RESYN_KEY_HOSTS, comma-separated. The person sets it; no tool argument can change it.""" return frozenset(name.strip().casefold() for name in os.environ.get("RESYN_KEY_HOSTS", "").split(",") if name.strip()) def key_host_allowed(url: str, key_hosts) -> bool: """May an API key go to this URL? Over https to any host, and over plain http only to this computer (localhost, 127.0.0.0/8 and ::1), where nothing crosses the network. key_hosts is the set of host names RESYN_KEY_HOSTS lists (exact match), for a caller that may name only those, as over MCP where an agent picks the endpoint; None on the command line, where the person typed the endpoint.""" if is_loopback(url): return True if urllib.parse.urlsplit(url).scheme.casefold() != "https": return False return key_hosts is None or _host(url) in key_hosts def _address(url: str) -> tuple[str, str, int | None]: """The scheme, host and port of a URL, with the scheme's default port filled in: one server, however the URL is written.""" parts = urllib.parse.urlsplit(url) scheme = parts.scheme.casefold() return scheme, _host(url), parts.port or {"https": 443, "http": 80}.get(scheme) def _key_refusal(key_name: str, base: str) -> str: """Why a key is not sent to an endpoint, in plain words. It names the variable and the host, never the key.""" host = _host(base) or "that endpoint" if urllib.parse.urlsplit(base).scheme.casefold() != "https" and not is_loopback(base): return (f"{key_name} is not sent over plain http to {host}: https is needed for any host but this computer. Use an https " "address for the endpoint") return (f"{key_name} is not sent to {host}: that is neither this computer nor a host in RESYN_KEY_HOSTS. Add the host name " "to RESYN_KEY_HOSTS (comma-separated, e.g. api.openai.com) to allow it, or use a model on this computer") def _token_refusal(site: str) -> str: """Why the site's token is not sent to an address, in plain words: plain http is for this computer alone, as it is for a key.""" host = _host(site) or "that site" return (f"RESYN_TOKEN is not sent over plain http to {host}: https is needed for any site but this computer. Use an https " "address for the site (RESYN_SITE)") def _probe(url: str, body: dict | None = None) -> dict | None: """GET a local server's info page, without any API key; with a body, a POST of that JSON. None when it is not there.""" try: status, data = _request(url, data=None if body is None else json.dumps(body).encode("utf-8"), timeout=5) except BenchError: return None return data if status == 200 and isinstance(data, dict) else None def detect_runtime(base: str) -> tuple[str | None, str | None]: """Which server answers at base, and its version: Ollama, llama.cpp, vLLM, LM Studio, or a hosted API by name. Hosted APIs are named by host and never probed. Local servers are asked their version without a key; llama.cpp's /props also holds the model's path on disk, so only its build_info is read. """ parts = urllib.parse.urlsplit(base) hosted = _known_host(HOSTED, base) if hosted: return hosted, None root = f"{parts.scheme}://{parts.netloc}" found = _probe(f"{root}/api/version") if found and isinstance(found.get("version"), str): return "Ollama", found["version"][:60] found = _probe(f"{root}/props") if found and ("build_info" in found or "default_generation_settings" in found): build = found.get("build_info") return "llama.cpp", build[:60] if isinstance(build, str) and build else None found = _probe(f"{root}/version") if found and isinstance(found.get("version"), str): return "vLLM", found["version"][:60] if parts.port == 1234 or _probe(f"{root}/api/v0/models") is not None: return "LM Studio", None return None, None def _llama_server_version(binary: str) -> str | None: """llama-server --version prints e.g. 'version: 5000 (abc1234)'.""" try: result = subprocess.run([binary, "--version"], capture_output=True, text=True, timeout=30) except (OSError, subprocess.SubprocessError): return None match = re.search(r"version:\s*(\d+)\s*\(([0-9a-f]+)\)", (result.stdout or "") + (result.stderr or "")) return f"b{match.group(1)}-{match.group(2)}" if match else None def _positive_int(value) -> int | None: return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else None def _ollama_name(name: str) -> str: """Ollama's name for a model: 'qwen3' means 'qwen3:latest'.""" return name if ":" in name else f"{name}:latest" def served_context(base: str, runtime: str | None, model: str) -> int | None: """The context Ollama or LM Studio loaded the model with, from the app's own info page (no key is sent). None when the page does not say, and for every other runtime: a llama-server's context is known from how it was started.""" parts = urllib.parse.urlsplit(base) root = f"{parts.scheme}://{parts.netloc}" kind = (runtime or "").casefold() # This runs after the run is done, so a page of another shape must cost only the context, never the run: anything that # is not a list of entries is read as saying nothing. if kind == "ollama": # The models loaded now, each with the context it was loaded with. models = (_probe(f"{root}/api/ps") or {}).get("models") for entry in models if isinstance(models, list) else []: name = entry.get("name") if isinstance(entry, dict) else None # Only text is read as a name; anything else is skipped as is. if isinstance(name, str) and _ollama_name(name) == _ollama_name(model): return _positive_int(entry.get("context_length")) # Not loaded now: Ollama loads it with the num_ctx its Modelfile sets, which /api/show names. Without one, the server's own # default applies, and this helper cannot know it, so nothing is said. parameters = (_probe(f"{root}/api/show", {"model": _ollama_name(model)}) or {}).get("parameters") # At most ten digits: a longer number is no context, and int() refuses a very long digit string (ValueError on Python 3.11+). found = re.search(r"(?m)^\s*num_ctx\s+(\d{1,10})\s*$", parameters) if isinstance(parameters, str) else None return _positive_int(int(found.group(1))) if found else None if kind == "lm studio": # Each model's loaded context length, once LM Studio has loaded it. models = (_probe(f"{root}/api/v0/models") or {}).get("data") for entry in models if isinstance(models, list) else []: if isinstance(entry, dict) and entry.get("id") == model: return _positive_int(entry.get("loaded_context_length")) return None def _key_part(value) -> str: """Must match resyn.tasksuite._key_part: how tool arguments are matched to the suite's fixed results.""" if isinstance(value, bool): return str(value).casefold() if isinstance(value, (int, float)): return f"{float(value):g}" text = str(value).strip().casefold() try: return f"{float(text):g}" except ValueError: return text def fixture_result(fixtures: list[dict], name: str, arguments) -> dict: fixture = next((f for f in fixtures if f.get("tool") == name), None) if fixture is None: return {"error": f"unknown tool {name}"} if not isinstance(arguments, dict): return {"error": "arguments must be an object"} key = "|".join(_key_part(arguments.get(k, "")) for k in fixture.get("keys", [])) return fixture.get("results", {}).get(key, fixture.get("default", {})) _BEARER = re.compile(r"(?i)\bbearer\s+[^\s\"'<>,;\[\]]+") # A token sent as Authorization: Bearer . _KEY_SHAPE = re.compile(r"\bsk-[A-Za-z0-9_-]{12,}") # An OpenAI-style key, even one the helper was not given. def _redact(text: str, api_key: str | None) -> str: """The text with the model's API key and any Bearer token taken out. An app can repeat a request's credentials in its error, and that error is posted to the site or printed in a report, so the key must never travel with it.""" if api_key: text = text.replace(api_key, "[key removed]") return _KEY_SHAPE.sub("[key removed]", _BEARER.sub("Bearer [key removed]", text)) def _words(value) -> str: """The message in an app's error: the text itself, or an object's message (or msg). A list gives its first message.""" if isinstance(value, list): value = value[0] if value else None if isinstance(value, dict): value = value.get("message", value.get("msg")) return value if isinstance(value, str) else "" def _has_markup(text: str) -> bool: """Is there an HTML tag in the text? A model, server or endpoint's name in angle brackets or quotes is the app's words, not a tag.""" return bool(_MARKUP.search(_WRAPPED_MODEL.sub("model", text))) def _is_page(raw: str) -> bool: """Is this reply markup from a proxy or a gateway, not the app's words? A tag of an HTML element anywhere in it makes it so, even inside the JSON message of an app's error: the words in such a message are not the app's either.""" return _has_markup(raw) def _app_words(raw: str, api_key: str | None = None) -> str: """The app's own message in an error reply, as one line: the message under error (or detail, as FastAPI-style apps send it), or the plain text, with the API key and any Bearer token removed. A reply with no words (an empty body, a web page) gives ''.""" try: body = json.loads(raw) except (ValueError, RecursionError): body = None if _is_page(raw) else raw.strip() if isinstance(body, list) and body and isinstance(body[0], dict) and "error" in body[0]: # [{"error": ...}], as some APIs send it body = body[0] if isinstance(body, dict): body = body["error"] if "error" in body else body.get("detail", body) # Anything a terminal would obey (an escape code, a bell) becomes a space, as a newline does: the words are printed and posted. return " ".join("".join(ch if ch.isprintable() else " " for ch in _redact(_words(body), api_key)).split()) def _app_error(raw: str, status: int, api_key: str | None = None) -> str: """The app's own words for a refusal, at most 200 characters, with the API key and any Bearer token removed. A reply with nothing a person can read (an empty body, a web page) gives the status instead.""" text = _app_words(raw, api_key) if not text: return f"HTTP {status}" return text if len(text) <= 200 else text[:197].rstrip() + "..." def _shown(value, api_key: str | None) -> str: """A reply as text for an error message: redacted and cut to 200 characters. A reply nested too deep to print is said so.""" try: return _redact(str(value), api_key)[:200] except RecursionError: return "a reply nested too deeply to show" def _is_refusal(status: int, raw: str) -> bool: """Is an app's error on a tools request the app saying it does not support the tools? Only a status in REFUSAL_STATUSES whose message matches REFUSAL_WORDS, with no markup in the reply or in the message once its JSON escapes are read, and no size or limit, and no name of one tool or option, in the message. Any other error is an error for the task, and nothing is posted for it.""" if status not in REFUSAL_STATUSES or _is_page(raw): return False message = _app_words(raw)[:2000] # Read after the JSON is decoded, so an escaped < is markup here too. A refusal is short; the cap keeps # the patterns' cost bounded on a huge error body (some of them are quadratic on one very long word). return (not _has_markup(message) and not _SIZED.search(message) and not _names_one(message) and bool(REFUSAL_WORDS.search(message))) class _KeyRedirects(urllib.request.HTTPRedirectHandler): """Follows a redirect only to the same address (scheme, host and port) the request went to, and only where the key may go. A redirect that moves the request elsewhere is refused before its request is made, for a model's key and the site's token alike: a key is never moved to another address or over plain http, not even to a host the person listed, because the address the person chose is the one that gets it.""" def __init__(self, key_hosts: frozenset[str] | None): self.key_hosts = key_hosts # As key_host_allowed takes it: None on the command line. def redirect_request(self, req, fp, code, msg, headers, newurl): if _address(newurl) != _address(req.full_url) or not key_host_allowed(newurl, self.key_hosts): parts = urllib.parse.urlsplit(newurl) where = f"{parts.scheme}://{parts.netloc.rpartition('@')[2]}" # The scheme and address only: no path, no user name. raise BenchError(f"The server redirected the request elsewhere, to {where}. A key is never moved to another address or over " "plain http, so the redirect was not followed") return super().redirect_request(req, fp, code, msg, headers, newurl) def _sender(url: str, key_hosts: frozenset[str] | None, keyed: bool): """The function that sends a request. One with no key goes through urlopen, as before. One with a key goes through an opener that follows a redirect only where _KeyRedirects allows it. To this computer it also skips any proxy: an HTTP_PROXY in the environment would otherwise be sent the key in the clear, though plain http to this computer never leaves it. Any other key goes over https (plain http is refused before a request), which a proxy only tunnels.""" if not keyed: return urllib.request.urlopen handlers = [_KeyRedirects(key_hosts)] if is_loopback(url): handlers.append(urllib.request.ProxyHandler({})) return urllib.request.build_opener(*handlers).open class Model: """One model behind an OpenAI-compatible /chat/completions endpoint. The API key goes only to that endpoint.""" def __init__(self, base: str, model: str, api_key: str | None = None, key_hosts: frozenset[str] | None = None): self.base, self.model, self.api_key = base.rstrip("/"), model, api_key self.key_hosts = key_hosts # Over MCP: the only hosts the key may go to. None: the command line, where the person named the endpoint. self.params: dict = {} self.reported_model: str | None = None self.usage: dict = {} def start_task(self) -> None: self.usage = {"prompt_tokens": 0, "completion_tokens": 0, "seconds": 0.0, "generating": 0.0, "measured": True, "counted": 0, "cut_off": False, "reported_tokens": 0, "reported_seconds": 0.0} def task_metrics(self) -> dict: """Tokens and time for the current task, over every request it made. cut_off is there only when the app stopped an answer at its length limit; generation_tps only when the app reported its own generation speed.""" u = self.usage metrics = {"seconds": round(u["seconds"], 3)} if u["measured"]: metrics.update(prompt_tokens=u["prompt_tokens"], completion_tokens=u["completion_tokens"]) if u["completion_tokens"] and u["generating"] > 0: metrics["tokens_per_second"] = round(u["completion_tokens"] / u["generating"], 2) if u["reported_seconds"] > 0: # Generation alone, without prompt processing: only llama.cpp's server reports it. metrics["generation_tps"] = round(u["reported_tokens"] / u["reported_seconds"], 2) elif u["counted"]: # Some replies came back with no count: the counted ones are kept, a floor, and give no speed. metrics.update(prompt_tokens=u["prompt_tokens"], completion_tokens=u["completion_tokens"], tokens_partial=True) if u["cut_off"]: metrics["cut_off"] = True return metrics def _record(self, data: dict, seconds: float) -> None: if not self.usage: return self.usage["seconds"] += seconds usage = data.get("usage") if isinstance(data.get("usage"), dict) else {} completion = usage.get("completion_tokens") if not isinstance(completion, int): self.usage["measured"] = False # The endpoint did not say for this reply; the others' counts are kept, as a floor. return self.usage["counted"] += 1 prompt = usage.get("prompt_tokens") self.usage["prompt_tokens"] += prompt if isinstance(prompt, int) else 0 self.usage["completion_tokens"] += completion # llama.cpp's server reports its own generation speed, without network or prompt time. Prefer it. timings = data.get("timings") if isinstance(data.get("timings"), dict) else {} rate = timings.get("predicted_per_second") if isinstance(rate, (int, float)) and not isinstance(rate, bool) and rate > 0 and completion: self.usage["generating"] += completion / rate self.usage["reported_tokens"] += completion self.usage["reported_seconds"] += completion / rate else: self.usage["generating"] += seconds def chat(self, messages: list[dict], tools: list | None = None) -> dict: """One request. No token limit is sent and no time limit is set: the model stops when it is done, or where its own server stops it, however long that takes.""" adapted = retried = 0 while True: payload = {"model": self.model, "messages": messages, **self.params} if tools: payload["tools"] = tools headers = {"Content-Type": "application/json", "User-Agent": USER_AGENT} if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" request = urllib.request.Request(f"{self.base}/chat/completions", data=json.dumps(payload).encode("utf-8"), headers=headers, method="POST") started = time.monotonic() send = _sender(self.base, self.key_hosts, keyed=bool(self.api_key)) # A key goes only where it may, and no redirect moves it. try: with send(request) as response: # No timeout: a slow answer is still an answer. data = _parse_json(response.read().decode("utf-8")) except urllib.error.HTTPError as exc: raw = exc.read().decode("utf-8", "replace") detail = raw[:500] if exc.code == 400 and adapted < 5 and self._adapt(detail): adapted += 1 continue if exc.code in RETRY_STATUS and retried < MAX_RETRIES: retried += 1 time.sleep(_wait_seconds(retried, exc.headers.get("Retry-After"))) continue # The app answered this tools request with an error of its own that says it will not run the tools: its words stand # in. Anything else is an error for the task (see _is_refusal). if tools and _is_refusal(exc.code, raw): raise Refused(_app_error(raw, exc.code, self.api_key)) from exc raise BenchError(f"The model's endpoint answered {exc.code}: {_redact(raw, self.api_key)[:500]}") from exc except (OSError, ValueError, http.client.HTTPException) as exc: # http.client quotes a header it will not send, key and all: the text is redacted before anything keeps it. text = _redact(str(exc), self.api_key) if isinstance(exc, OSError): raise Unreachable(f"Could not reach the model at {self.base}: {text}") from None raise BenchError(f"Could not get an answer from the model at {self.base}: {text}") from None if not isinstance(data, dict): # Valid JSON, but not a chat reply. raise BenchError(f"The endpoint returned no message: {_shown(data, self.api_key)}") self._record(data, time.monotonic() - started) reported = data.get("model") # Posted as text, so only a string is kept. self.reported_model = self.reported_model or (reported if isinstance(reported, str) else None) choices = data.get("choices") first = choices[0] if isinstance(choices, list) and choices else None message = first.get("message") if isinstance(first, dict) else None if not isinstance(message, dict): raise BenchError(f"The endpoint returned no message: {_shown(data, self.api_key)}") if first.get("finish_reason") == "length" and self.usage: self.usage["cut_off"] = True return message def _adapt(self, detail: str) -> bool: """Some hosted models refuse fixed sampling settings. Drop what they refuse, and remember it.""" text = detail.casefold() changed = False for name in ("temperature", "top_p", "seed"): if name in text and name in self.params: self.params.pop(name) changed = True return changed def _answer_tool_task(model: Model, task: dict, max_turns: int) -> list[dict]: messages = ([{"role": "system", "content": task["system"]}] if task.get("system") else []) + \ [{"role": "user", "content": task["prompt"]}] counter = 0 for _ in range(max_turns): reply = model.chat(messages, task.get("tools")) calls = reply.get("tool_calls") or [] message = {"role": "assistant", "content": reply.get("content")} if calls: fixed = [] for call in calls: counter += 1 fixed.append({"id": str(call.get("id") or f"call_{counter}"), "type": "function", "function": {"name": (call.get("function") or {}).get("name"), "arguments": (call.get("function") or {}).get("arguments") or "{}"}}) message["tool_calls"] = fixed messages.append(message) if not calls: return messages for call in message["tool_calls"]: raw = call["function"]["arguments"] try: arguments = json.loads(raw) if isinstance(raw, str) else raw except ValueError: arguments = None result = fixture_result(task.get("fixtures", []), call["function"]["name"], arguments) messages.append({"role": "tool", "tool_call_id": call["id"], "content": json.dumps(result)}) return messages _GPU_NOISE = re.compile(r"(?i)\b(?:nvidia|geforce|amd|radeon|graphics|gpu)\b|\(r\)|\(tm\)") _CPU_NOISE = re.compile(r"(?i)\b(?:amd|intel|core\(tm\)|processor|cpu)\b|\(r\)|\(tm\)|\d+-core|@\s*[\d.]+\s*ghz") def _short(raw: str, noise) -> str: """'NVIDIA GeForce RTX 3060' -> 'RTX 3060'; 'AMD Ryzen 5 5600X 6-Core Processor' -> 'Ryzen 5 5600X'. Matches the site's run reports.""" return re.sub(r"\s+", " ", noise.sub(" ", raw.split(",")[0])).strip(" ,-") def _hardware_label(hardware: dict) -> str | None: """One line in the form run reports use: 'RTX 3060 12 GB', 'M3 Max 36 GB', 'Ryzen 5 5600X, 32 GB'.""" if hardware.get("kind") == "gpu" and hardware.get("gpu"): return f"{_short(hardware['gpu'], _GPU_NOISE)}" + (f" {hardware['vram']:g} GB" if hardware.get("vram") else "") if hardware.get("kind") == "apple": chip = re.sub(r"(?i)^apple\s+", "", hardware.get("chip") or "") or "Apple silicon" return f"{chip}" + (f" {hardware['memory']:g} GB" if hardware.get("memory") else "") if hardware.get("kind") == "cpu" or hardware.get("ram"): name = _short(hardware.get("cpu") or "", _CPU_NOISE) or "CPU" return f"{name}, {hardware['ram']:g} GB" if hardware.get("ram") else name return None def _setup_fields(hardware: dict) -> dict | None: """The computer as the site's hardware picker names it, which a run needs to rank; None when something is missing.""" gb = lambda value: f"{value:g}" if value else "" if hardware.get("kind") == "apple" and hardware.get("chip") and hardware.get("memory"): return {"kind": "apple", "apple_chip": re.sub(r"(?i)^apple\s+", "", hardware["chip"]), "unified_memory_gb": gb(hardware["memory"])} if hardware.get("kind") == "gpu" and hardware.get("gpu") and hardware.get("vram") and hardware.get("ram"): name = hardware["gpu"] vendor = ("NVIDIA" if re.search(r"(?i)nvidia|geforce|rtx|gtx|tesla|quadro", name) else "AMD" if re.search(r"(?i)amd|radeon|\brx\b", name) else "Intel" if re.search(r"(?i)intel|\barc\b", name) else "Other") return {"kind": "gpu", "gpu_vendor": vendor, "gpu_model": name, "gpu_vram_gb": gb(hardware["vram"]), "gpu_system_ram_gb": gb(hardware["ram"])} if hardware.get("kind") == "cpu" and hardware.get("ram"): return {"kind": "cpu", "cpu_model": hardware.get("cpu") or "CPU", "cpu_system_ram_gb": gb(hardware["ram"])} return None def resolve_hardware(overrides: dict | None, detect: bool) -> tuple[str | None, list[str], dict | None]: """The hardware line to post, notes about flags that did not apply, and the picker fields for ranking. detect: describe this machine. Otherwise only what the caller stated is used, since this machine is not the one serving the model. """ given = {k: v for k, v in (overrides or {}).items() if v not in (None, "")} merged = {**(detect_hardware() if detect else {}), **given} if "kind" not in merged and "gpu" in given: merged["kind"] = "gpu" if merged.get("kind") == "apple" and "ram" in given: merged["memory"] = given["ram"] used = {"gpu": {"gpu", "vram"}, "apple": {"ram"}, "cpu": {"ram"}}.get(merged.get("kind"), set()) notes = [f"--{key} was not used: it does not apply to {merged.get('kind') or 'this'} hardware" for key in ("gpu", "vram", "ram") if key in given and key not in used] label = _hardware_label(merged) if given and label is None: notes.append("no hardware line could be made from --kind, --gpu, --vram and --ram; the run carries none") return label, notes, _setup_fields(merged) def _free_port() -> int: with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) return sock.getsockname()[1] def _start_llama_server(path: Path, server: str | None, extra: list[str]): """Start llama-server and wait until it answers /health, however long the model takes to load; a server that exits meanwhile is ServerDied.""" binary = server or shutil.which("llama-server") if not binary: raise BenchError("llama-server was not found. Install llama.cpp and pass its path (--llama-server, or llama_server over MCP), or name an endpoint") port = _free_port() command = [binary, "-m", str(path), "--host", "127.0.0.1", "--port", str(port), "--jinja", *extra] process = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) base = f"http://127.0.0.1:{port}" while True: if process.poll() is not None: raise ServerDied(f"llama-server stopped while loading the model (exit {process.returncode})") try: with urllib.request.urlopen(f"{base}/health", timeout=5) as response: if response.status == 200: return process, f"{base}/v1", _llama_server_version(binary) except OSError: pass time.sleep(1) def _sets_context(args: list[str]) -> bool: """Does the caller set llama-server's context, with -c or --ctx-size in any of its forms?""" return any(a in ("-c", "--ctx-size") or a.startswith(("-c=", "--ctx-size=")) for a in args) def _given_context(args: list[str]) -> int | None: """The context the caller set with -c or --ctx-size, in tokens. The last one wins, as it does for llama-server. None when the value is not a number above zero: 0 asks llama-server for the model's own context, which this helper cannot know.""" value = None for index, arg in enumerate(args): if arg in ("-c", "--ctx-size") and index + 1 < len(args): value = args[index + 1] elif arg.startswith(("-c=", "--ctx-size=")): value = arg.partition("=")[2] try: number = int(value) except (TypeError, ValueError): return None return number if number > 0 else None def _checked_size(size) -> str: """The size as named, when it is one of SIZES. BenchError, in words, otherwise.""" if size not in SIZES: raise BenchError(f"size must be {SIZE_WORDS}") return size def _variant_size(variant) -> str | None: """The size a sized variant was issued for, which a post names; None for a variant that names no size.""" match = _SIZED_VARIANT.fullmatch(variant) if isinstance(variant, str) else None return match.group(1) if match else None def _is_unscored(task: dict) -> bool: """A task the suite marks ranked: false. It is answered and posted like any other, but no points, pass or score counts it.""" return task.get("ranked") is False def run_task_suite(*, site: str = SITE, path: str | Path | None = None, provider: str | None = None, endpoint: str | None = None, model: str | None = None, api_key_env: str | None = None, agent: str | None = None, token: str | None = None, only: list[str] | None = None, server: str | None = None, server_args: list[str] | None = None, model_url: str | None = None, runtime: str | None = None, runtime_version: str | None = None, overrides: dict | None = None, dry_run: bool = False, guard_keys: bool = False, size: str = DEFAULT_SIZE, progress=None) -> dict: """Ask a model the tasks of one size of the suite, the scored tasks first and then any the suite does not score, and post its answers as that size. The site grades them. guard_keys: the MCP tool's rule. An API key from an environment variable goes only over https, and only to this computer or to a host in RESYN_KEY_HOSTS, and anything else is refused before any request. The command line, where the person types the endpoint, is not limited to listed hosts, but it still sends a key only over https or to this computer. size: which of the suite's sizes to answer, fast, medium or full (DEFAULT_SIZE when none is named). """ site = site.rstrip("/") size = _checked_size(size) if token and not key_host_allowed(site, None): # Checked before the suite is fetched or a model is asked: never plain http. raise BenchError(_token_refusal(site)) status, suite = _request(f"{site}/api/suites/{TASK_SUITE}?size={size}") if status != 200: raise BenchError(f"Could not load the task suite ({status})") tasks = [t for t in suite["tasks"] if not only or t["id"] in only or t["category"] in only] if not tasks: raise BenchError("No tasks match --only") # The unscored tasks go after every scored task, each group in the suite's order (sorted is stable), so a run stopped early keeps its scored answers. tasks = sorted(tasks, key=_is_unscored) subject: dict = {} notes: list[str] = [] process = None server_context = None # The context llama-server runs with, when it is known: the one the helper chose, or the caller's own -c. picked_context = None # The one the helper chose, and no -c the caller gave: the params carry only this, as they always did. file_path = Path(path).expanduser() if path else None if file_path is not None: if not file_path.is_file(): raise BenchError(f"No such file: {file_path}") subject = {"sha256": sha256_of(file_path), "model": file_path.stem[:120]} preset_base, preset_key = ENDPOINTS.get(provider or "", (None, None)) base = endpoint or preset_base try: if base is None: if file_path is None: raise BenchError("Name a model endpoint: --provider openai|openrouter|ollama|lmstudio|llamacpp, or --endpoint URL") extra = list(server_args or []) if _sets_context(extra): process, base, started_version = _start_llama_server(file_path, server, extra) server_context = _given_context(extra) # The caller's own -c, recorded as the run's context when it is a number. else: # No answer is capped, so the context is the only limit: as large as fits. A server that dies while # loading did not fit; try smaller before giving up. for context in CONTEXTS: try: process, base, started_version = _start_llama_server(file_path, server, ["-c", str(context)] + extra) break except ServerDied as exc: if context == CONTEXTS[-1]: raise if progress: progress(f"note: llama-server did not start with a {context}-token context ({exc}); trying smaller") else: # pragma: no cover raise BenchError("llama-server did not start") server_context = picked_context = context if context != CONTEXTS[0]: notes.append(f"llama-server runs with a {context}-token context, the largest that fit; long answers stop there") provider = "local" if not runtime: runtime, runtime_version = "llama.cpp", runtime_version or started_version local = is_loopback(base) if file_path is not None and not local: notes.append(f"{base} is not on this computer, so the site records the run for {file_path.name} without checking " "that the endpoint serves it. This computer's hardware is left out; --kind, --gpu, --vram and --ram describe the server") hardware, hardware_notes, setup = resolve_hardware(overrides, detect=local) notes.extend(hardware_notes) for note in notes: if progress: progress(f"note: {note}") key_name = api_key_env or preset_key if key_name and key_name.upper() == "RESYN_TOKEN": raise BenchError("RESYN_TOKEN is your resyn account token and is never sent to a model. Use the model provider's own key") api_key = _credential(os.environ.get(key_name) or "", key_name) if key_name else None # Checked before any header is built. if key_name and not api_key: raise BenchError(f"Set {key_name} to your API key for {provider or base}") # The key is checked against its endpoint before any request goes out, the endpoint's own detection included: over https, and over # MCP also to a listed host. The command line has no list, so it is checked for https alone. key_hosts = _listed_key_hosts() if guard_keys else None if api_key and not key_host_allowed(base, key_hosts): raise BenchError(_key_refusal(key_name, base)) if not runtime: runtime, detected_version = detect_runtime(base) runtime_version = runtime_version or detected_version if not model and file_path is None: raise BenchError("Name the model with --model (as the endpoint knows it, e.g. openai/gpt-4o-mini)") llm = Model(base, model or "local", api_key, key_hosts=key_hosts) llm.params = dict(suite.get("params") or {}) answers, refused, errors, metrics, cut_off = {}, {}, {}, {}, [] # refused: task id -> the app's error, for tool tasks it refused first_unscored = next((number for number, task in enumerate(tasks, 1) if _is_unscored(task)), None) for number, task in enumerate(tasks, 1): if progress: if number == first_unscored: progress("these tasks are not scored: they are answered and posted, but no score counts them") progress(f"[{number}/{len(tasks)}] {task['id']} {task['title']}" + (" (not scored)" if _is_unscored(task) else "")) llm.start_task() try: if task["kind"] == "tools": answers[task["id"]] = _answer_tool_task(llm, task, int(task.get("max_tool_turns") or suite.get("max_tool_turns", 6))) else: messages = ([{"role": "system", "content": task["system"]}] if task.get("system") else []) + \ [{"role": "user", "content": task["prompt"]}] answers[task["id"]] = llm.chat(messages).get("content") or "" except Unreachable: raise # The site grades a posted refusal as failed and keeps the app's words with the run (docs/operations.md, Task suite). except Refused as exc: # The app would not run the tools, so its own words stand in for the answer. answers[task["id"]] = {"refused": exc.reason} refused[task["id"]] = exc.reason if progress: progress(f" refused by the app: {exc.reason}") except BenchError as exc: errors[task["id"]] = str(exc) if progress: progress(f" failed: {exc}") if task["id"] in answers: # Answered or refused: the tokens and time are what the app used either way. metrics[task["id"]] = llm.task_metrics() if llm.usage["cut_off"]: cut_off.append(task["id"]) if progress: progress(" cut off by the server's own limit (finish_reason length)") # llama-server's context is known from how it was started. Ollama and LM Studio say what they loaded in their info pages, read # after the run is done: a page of another shape costs the context, never the run. try: context_tokens = server_context or served_context(base, runtime, llm.model) except (BenchError, ValueError, TypeError, RecursionError): context_tokens = None finally: if process is not None: process.terminate() try: process.wait(timeout=30) except subprocess.TimeoutExpired: process.kill() name = (model or subject["model"]) if subject else model answered = len(answers) - len(refused) # A task the app refused is posted, but it is not an answer the model gave. body = {"suite": TASK_SUITE, "size": size, "variant": suite.get("variant"), **subject, "answers": answers, "metrics": metrics, "mode": "endpoint", "agent": agent, "hardware": hardware, "setup": setup, **({"model_url": model_url} if model_url else {}), "endpoint_model": (llm.reported_model or "")[:160] or None, "harness": HARNESS, "endpoint": base, # The address the model answered at: a hosted service's marks the run hosted (resyn/hosting.py). "runtime": (runtime or "")[:80] or None, "runtime_version": (runtime_version or "")[:60] or None, # The sampling settings, and the context the helper picked for llama-server, as they always were. The context is also posted at # the top, which is where the contract names it. "params": {**llm.params, **({"context": picked_context} if picked_context else {})}, **({"context": context_tokens} if context_tokens else {})} if subject: body["model"] = name[:120] else: body.update(model=name, provider=provider if provider in ENDPOINTS else _known_host(HOST_PROVIDERS, base) or "other") generated = sum(m.get("completion_tokens", 0) for m in metrics.values()) read = sum(m.get("prompt_tokens", 0) for m in metrics.values()) # The printed summary keeps its keys. refused and context are added only when a run has one to report. summary = {"size": size, "model": name, "endpoint": base, "runtime": body["runtime"], "runtime_version": body["runtime_version"], "harness": HARNESS, "answered": answered, "errors": errors, "params": body["params"], "completion_tokens": generated, "prompt_tokens": read, "tokens": read + generated, "seconds": round(sum(m.get("seconds", 0) for m in metrics.values()), 1), "cut_off": cut_off, "notes": notes, **({"refused": refused} if refused else {}), **({"context": context_tokens} if context_tokens else {}), **({"tokens_partial": True} if any(m.get("tokens_partial") for m in metrics.values()) else {})} if cut_off: notes.append(f"{len(cut_off)} task(s) were cut off by the server's own limit ({', '.join(cut_off)}): its context or output cap. " "Raise it and run again: Ollama's num_ctx, llama-server's -c (--server-arg=-c --server-arg=65536), or a hosted " "model with a larger output limit") # A run the app refused in full is posted in the refused form: that is what the app said. Nothing answered at all is not posted. if not answers and not dry_run: notes.append("Nothing was answered, so nothing is posted: each task's error is listed under errors") if dry_run or not answers: return {**summary, "posted": False, "answers": answers} result = _post_run(site, body, token) return {**summary, "posted": True, "url": result["url"], "passed": result["passed"], "total": result["total"], "categories": result["categories"], "tokens_per_second": result.get("tokens_per_second"), "ranking": result.get("ranking")} def _post_run(site: str, body: dict, token: str | None) -> dict: """Post a finished run. If that fails, keep the answers in a file so a long run is not lost.""" try: status, result = _request(f"{site}/api/suite-runs", data=json.dumps(body).encode("utf-8"), token=token, timeout=120) if status == 201: return result detail = result.get("error") if isinstance(result, dict) and result.get("error") else result # Any reply shape: the fallback must run. reason = f"The site did not accept the run ({status}): {detail}" except BenchError as exc: reason = str(exc) reason = _redact(reason, token) # The site may echo the request back; the token never travels in a message. saved = Path(f"resyn-run-{time.strftime('%Y%m%d-%H%M%S')}.json") try: saved.write_text(json.dumps(body), encoding="utf-8") except OSError as exc: raise BenchError(f"{reason}. The answers could not be saved either: {exc}") from exc raise BenchError(f"{reason}. The answers are saved in {saved.resolve()}; post them again with --post-file {saved}") def post_file(path: str | Path, *, site: str = SITE, token: str | None = None) -> dict: """Post a run that an earlier attempt saved because the site could not take it.""" try: body = json.loads(Path(path).expanduser().read_text(encoding="utf-8")) except (OSError, ValueError) as exc: raise BenchError(f"Could not read {path}: {exc}") from exc if not isinstance(body, dict) or "answers" not in body: raise BenchError(f"{path} is not a saved run") result = _post_run(site.rstrip("/"), body, token) return {"posted": True, **{k: result.get(k) for k in ("url", "passed", "total", "categories", "tokens", "tokens_per_second")}} def post_own_answers(*, site: str = SITE, model: str, provider: str | None, answers: dict, agent: str | None, token: str | None, harness: str | None = None, variant: str | None = None) -> dict: """An agent that answered the tasks itself posts them here. Labelled as answered directly by an agent. variant: the suite's variant field, as fetched; the reasoning, code and tool tasks change with it. Its size is the size the post names: the one the variant was issued for, so the answers must be the tasks of that size. harness: what drove the model, e.g. 'Claude Code'; the site falls back to the agent's name. """ body = {"suite": TASK_SUITE, "size": _variant_size(variant), "model": model, "provider": provider or "other", "answers": answers, "mode": "self", "agent": agent, "harness": harness or agent, "variant": variant} status, result = _request(f"{site.rstrip('/')}/api/suite-runs", data=json.dumps(body).encode("utf-8"), token=token, timeout=120) if status != 201: # The site's reply is read as an app's error is, so an echoed token never reaches the message. raise BenchError(f"The site did not accept the run ({status}): {_app_error(json.dumps(result), status, token)}") return result # --- MCP: the same thing as tools, over stdio -------------------------------------------------------------------- TOOLS = [ { "name": "resyn_lookup_file", "description": "The SHA-256 of a local GGUF file.", "inputSchema": {"type": "object", "properties": {"path": {"type": "string", "description": "Path to the .gguf file"}}, "required": ["path"]}, }, { "name": "resyn_run_task_suite", "description": ("Run the resyn task suite (summarizing, extracting, instructions, reasoning, reading code, tool use " "and building web pages) on a model and post its answers; the site grades them. The size is fast " "when none is named. Give either a local GGUF path (started with llama-server) or a provider/endpoint " "and model name. API keys are read from environment variables. A key goes only over https, and only " "to this computer or to a host listed in the RESYN_KEY_HOSTS environment variable (plain http is for " "this computer alone). Every task of the size is answered: the scored tasks in order, then any the " "suite does not score."), "inputSchema": {"type": "object", "properties": { "path": {"type": "string", "description": "A GGUF file to run locally"}, "provider": {"type": "string", "enum": sorted(ENDPOINTS), "description": "A known endpoint"}, "endpoint": {"type": "string", "description": "Any OpenAI-compatible base URL, e.g. http://127.0.0.1:8080/v1"}, "model": {"type": "string", "description": "The model's name at that endpoint"}, "api_key_env": {"type": "string", "description": "Name of the environment variable holding the endpoint's API key. Only names ending in API_KEY are accepted"}, "only": {"type": "array", "items": {"type": "string"}, "description": "Task ids or categories to run (default: all)"}, "model_url": {"type": "string", "description": "Where people can get the same model: its Hugging Face page or any other https link"}, "llama_server": {"type": "string", "description": "Path to llama-server, for a local file with no endpoint (its file name must be llama-server, llama-server.exe, or llama-server with a backend suffix such as -cuda or -vulkan)"}, "server_args": {"type": "array", "items": {"type": "string"}, "description": "Extra llama-server options, e.g. [\"-ngl\", \"99\"]. Only -c or --ctx-size, -ngl or --n-gpu-layers, -t or --threads, -b or --batch-size, -ub or --ubatch-size, -fa or --flash-attn, and --jinja are allowed; any other is refused before a server starts. -c defaults to the largest of 32768, 16384 and 8192 that fits"}, "runtime": {"type": "string", "description": "What serves the model, e.g. llama.cpp, Ollama, vLLM (detected when left out)"}, "runtime_version": {"type": "string", "description": "That server's version (detected when left out)"}, "dry_run": {"type": "boolean", "description": "Ask the model but do not post"}, "size": {"type": "string", "enum": list(SIZES), "description": "The size of the suite to run: fast (the default), medium or full. Each size is a set of the suite's tasks, and the run is posted as that size"}, }}, }, { "name": "resyn_get_task_suite", "description": ("Get one size of the resyn task suite's prompts, tools and fixtures, to answer the tasks yourself. " "The size is fast when none is named. Post the answers with the variant this returns."), "inputSchema": {"type": "object", "properties": { "size": {"type": "string", "enum": list(SIZES), "description": "The size of the suite to fetch: fast (the default), medium or full"}, }}, }, { "name": "resyn_post_task_answers", "description": ("Post your own answers to the resyn task suite, labelled as answered directly by an agent. answers " "maps task id to your reply text; for tool tasks, to the whole conversation as OpenAI chat " "messages, with tool results taken from the task's fixtures. The post is of the size its variant " "was fetched for."), "inputSchema": {"type": "object", "properties": { "model": {"type": "string", "description": "Your model's name"}, "provider": {"type": "string", "description": "Who serves you, e.g. anthropic, openai"}, "harness": {"type": "string", "description": "The program driving you, e.g. Claude Code, Codex CLI, Cursor"}, "answers": {"type": "object"}, "variant": {"type": "string", "description": "The variant field of the suite you answered, as resyn_get_task_suite gave it"}, }, "required": ["model", "answers", "variant"]}, }, ] _MCP_KEY_NAME = re.compile(r"^[A-Z0-9_]*API_KEY$") # Not GITHUB_TOKEN and the like: only model API keys. # llama-server, with a backend suffix at most (llama-server-cuda, llama-server-vulkan) and the .exe of Windows. The suffixes are a closed list: # a name that only starts with llama-server, such as llama-server-evil, is a program of any kind and not a server, so it is refused. _SERVER_BINARY = re.compile(r"llama-server(?:-(?:cpu|cuda\d*|hip|rocm\d*|vulkan|metal|sycl|opencl|blas|kompute|musa|cann|avx512|avx2|avx))?" r"(?:\.exe)?", re.IGNORECASE) def _string_list(arguments: dict, field: str) -> list[str] | None: value = arguments.get(field) if value is None: return None if not isinstance(value, list) or not all(isinstance(item, str) for item in value): raise BenchError(f"{field} must be a list of strings") return value # The llama-server options an agent may pass over MCP, with the value each takes (a pattern, and an example for the error), or None for a # switch. Each sets how the model runs: its context, GPU layers, threads, batch sizes, attention or chat template. None of them moves the # server, changes its model or opens it to the network, which --host, --port, -m, --api-key, --path and --listen would do. _MCP_SERVER_OPTIONS = { "-c": (r"\d+", "4096"), "--ctx-size": (r"\d+", "4096"), "-ngl": (r"-?\d+|auto|all", "99"), "--n-gpu-layers": (r"-?\d+|auto|all", "99"), "-t": (r"\d+", "8"), "--threads": (r"\d+", "8"), "-b": (r"\d+", "512"), "--batch-size": (r"\d+", "512"), "-ub": (r"\d+", "128"), "--ubatch-size": (r"\d+", "128"), "-fa": (r"on|off|auto", "on"), "--flash-attn": (r"on|off|auto", "on"), "--jinja": None, } _MCP_OPTIONAL_VALUE = ("-fa", "--flash-attn") # llama-server turns flash attention on when it is given with no value. _MCP_SERVER_ALLOWED = ("-c or --ctx-size (the context), -ngl or --n-gpu-layers (GPU layers), -t or --threads (threads), " "-b or --batch-size and -ub or --ubatch-size (batch sizes), -fa or --flash-attn (flash attention), " "and --jinja (the chat template)") def _check_server_args(args: list[str] | None) -> None: """Over MCP an agent may pass only the llama-server options above, each with a value that suits it. Anything else is refused here, before a server starts: the helper sets the server's address and its model itself, and an option that moved either, or opened the server to the network, would undo that. The command line is not limited.""" tokens = list(args or []) index = 0 while index < len(tokens): name, equals, inline = tokens[index].partition("=") index += 1 if name not in _MCP_SERVER_OPTIONS: raise BenchError(f"{name[:40] or 'An empty argument'} is not a server option an agent may pass over MCP, so no server was " f"started. The options allowed are: {_MCP_SERVER_ALLOWED}.") option = _MCP_SERVER_OPTIONS[name] if option is None: # A switch, which takes no value. if equals: raise BenchError(f"{name} takes no value, so no server was started") continue pattern, example = option if equals: # --ctx-size=4096 if re.fullmatch(pattern, inline) is None: raise _needs_value(name, example) elif index < len(tokens) and re.fullmatch(pattern, tokens[index]) is not None: # -ngl 99 index += 1 elif name not in _MCP_OPTIONAL_VALUE: raise _needs_value(name, example) def _needs_value(name: str, example: str) -> BenchError: return BenchError(f"{name} needs a value that suits it, for example {name} {example}, so no server was started") def _call_tool(name: str, arguments: dict, defaults: dict) -> dict: if not isinstance(arguments, dict): raise BenchError("arguments must be an object") path = arguments.get("path") or "" if name == "resyn_lookup_file": file = Path(path).expanduser() if not file.is_file(): raise BenchError(f"No such file: {file}") return {"sha256": sha256_of(file)} if name == "resyn_run_task_suite": key_name = arguments.get("api_key_env") # An agent picks this name: never let it point at the resyn token. if key_name is not None and (not isinstance(key_name, str) or not _MCP_KEY_NAME.match(key_name) or key_name == "RESYN_TOKEN"): raise BenchError("api_key_env must name a variable ending in API_KEY") server = arguments.get("llama_server") if server is not None and (not isinstance(server, str) or _SERVER_BINARY.fullmatch(Path(server).name) is None): raise BenchError("llama_server must be the path to llama-server: a file named llama-server, llama-server.exe, or llama-server with " "a backend suffix such as -cuda or -vulkan") server_args = _string_list(arguments, "server_args") _check_server_args(server_args) # Refused before any server starts: an agent may pass only the options listed above. return run_task_suite(site=defaults["site"], path=arguments.get("path"), provider=arguments.get("provider"), endpoint=arguments.get("endpoint"), model=arguments.get("model"), api_key_env=arguments.get("api_key_env"), agent=defaults["agent"], token=defaults["token"], only=_string_list(arguments, "only"), server=server, model_url=arguments.get("model_url") if isinstance(arguments.get("model_url"), str) else None, server_args=server_args, runtime=arguments.get("runtime"), runtime_version=arguments.get("runtime_version"), dry_run=bool(arguments.get("dry_run")), size=_checked_size(arguments.get("size", DEFAULT_SIZE)), guard_keys=True) # An agent picks the endpoint, so the key goes only where RESYN_KEY_HOSTS allows. if name == "resyn_get_task_suite": size = _checked_size(arguments.get("size", DEFAULT_SIZE)) status, suite = _request(f"{defaults['site']}/api/suites/{TASK_SUITE}?size={size}") if status != 200: raise BenchError(f"Could not load the task suite ({status})") return suite if name == "resyn_post_task_answers": return post_own_answers(site=defaults["site"], model=arguments.get("model") or "", provider=arguments.get("provider"), answers=arguments.get("answers") or {}, agent=defaults["agent"], token=defaults["token"], harness=arguments.get("harness"), variant=arguments.get("variant")) raise BenchError(f"Unknown tool {name}") def serve_mcp(defaults: dict, stdin=sys.stdin, stdout=sys.stdout) -> None: """A minimal MCP server (JSON-RPC 2.0, one message per line) exposing the tools above.""" def reply(message_id, result=None, error=None): message = {"jsonrpc": "2.0", "id": message_id} message.update({"error": error} if error else {"result": result}) stdout.write(json.dumps(message) + "\n") stdout.flush() for line in stdin: try: message = json.loads(line) except (ValueError, RecursionError): # A message nested too deep to parse is skipped, as a garbled one is. continue if not isinstance(message, dict): continue method, message_id, params = message.get("method"), message.get("id"), message.get("params") or {} if message_id is None: # A notification, such as notifications/initialized: nothing to answer. continue if method == "initialize": client = params.get("clientInfo") or {} if not defaults["agent"] and client.get("name"): defaults["agent"] = f"{client['name']} {client.get('version', '')}".strip()[:80] reply(message_id, {"protocolVersion": params.get("protocolVersion") or "2025-06-18", "capabilities": {"tools": {}}, "serverInfo": {"name": "resyn-bench", "version": "1"}}) elif method == "ping": reply(message_id, {}) elif method == "tools/list": reply(message_id, {"tools": TOOLS}) elif method == "tools/call": try: result = _call_tool(params.get("name", ""), params.get("arguments") or {}, defaults) reply(message_id, {"content": [{"type": "text", "text": json.dumps(result, indent=2)}], "isError": False}) except Exception as exc: # One bad call must not end the server. reply(message_id, {"content": [{"type": "text", "text": str(exc) or type(exc).__name__}], "isError": True}) else: reply(message_id, error={"code": -32601, "message": f"Unknown method {method}"}) def _token_from_environment() -> str | None: """RESYN_TOKEN as set. A blank one is no token at all: the run posts without an account, and the person is told so.""" value = os.environ.get("RESYN_TOKEN") if value is not None and not value.strip(): print("RESYN_TOKEN is set but blank, so this run will post without an account. Set it to your token from your profile page " "to post under your name.", file=sys.stderr) return None return value or None def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Run the resyn task suite on a model and post its answers to resyn.cloud. The size picks " "the tasks: fast unless --size says medium or full.") parser.add_argument("file", nargs="?", help="a .gguf file: started with llama-server, or named as what the endpoint serves") parser.add_argument("--agent", help="name shown as the agent that posted it (e.g. 'Claude Code')") parser.add_argument("--site", default=os.environ.get("RESYN_SITE", SITE), help=argparse.SUPPRESS) parser.add_argument("--tasks", action="store_true", help=argparse.SUPPRESS) # The suite is all it runs now; kept for old commands. parser.add_argument("--provider", choices=sorted(ENDPOINTS), help="a known endpoint (its API key comes from the usual variable)") parser.add_argument("--endpoint", help="any OpenAI-compatible base URL, e.g. http://127.0.0.1:8080/v1") parser.add_argument("--model", help="the model's name at the endpoint") parser.add_argument("--api-key-env", help="environment variable holding the endpoint's API key") parser.add_argument("--only", action="append", help="a task id or category to run; repeat for more (default: all)") parser.add_argument("--size", choices=SIZES, default=DEFAULT_SIZE, help="which size of the suite to run: fast (the default), medium or full. Each size is a set of the suite's " "tasks; the run answers that set and is posted as that size") parser.add_argument("--model-url", help="where people can get the same model: its Hugging Face page or any other https link") parser.add_argument("--min-max-tokens", type=int, default=0, help=argparse.SUPPRESS) # Gone: no answer is capped. Accepted so old commands still run. parser.add_argument("--timeout", type=float, default=0, help=argparse.SUPPRESS) # Gone: an answer is waited for as long as it takes. parser.add_argument("--llama-server", help="path to llama-server, for a local file with no endpoint") parser.add_argument("--server-arg", action="append", default=None, help="extra llama-server argument; repeat for more. Write a dash value as --server-arg=-ngl") parser.add_argument("--post-file", metavar="FILE", help="post a run an earlier attempt saved because the site could not take it") parser.add_argument("--runtime", help="what serves the model, e.g. vLLM or 'llama.cpp' (detected when left out)") parser.add_argument("--runtime-version", help="that server's version (detected when left out)") parser.add_argument("--kind", choices=("gpu", "apple", "cpu"), help="override the detected hardware kind") parser.add_argument("--gpu", help="override the GPU name") parser.add_argument("--vram", type=float, help="GPU memory in GB") parser.add_argument("--ram", type=float, help="system memory in GB") parser.add_argument("--dry-run", action="store_true", help="ask the model, show what would be posted, post nothing") parser.add_argument("--mcp", action="store_true", help="serve as an MCP tool server on stdin/stdout. A key goes only over https, and only to this computer " "or to a host in RESYN_KEY_HOSTS") args = parser.parse_args(argv) if not (args.mcp or args.post_file) and (args.file or args.provider or args.endpoint): print(f"Running the {args.size} size of {TASK_SUITE}.", file=sys.stderr, flush=True) # The first line a run prints. if args.min_max_tokens: print("note: --min-max-tokens does nothing now; no answer is capped", file=sys.stderr) if args.timeout: print("note: --timeout does nothing now; every answer is waited for as long as it takes", file=sys.stderr) token = _token_from_environment() if args.mcp: serve_mcp({"site": args.site.rstrip("/"), "agent": args.agent, "token": token}) return 0 if args.post_file: try: print(json.dumps(post_file(args.post_file, site=args.site, token=token), indent=2)) except BenchError as exc: print(json.dumps({"error": str(exc)}), file=sys.stderr) return 1 return 0 if not args.file and not args.provider and not args.endpoint: parser.error("name a .gguf file, or a model with --provider/--endpoint and --model, or use --mcp") try: result = run_task_suite(site=args.site, path=args.file, provider=args.provider, endpoint=args.endpoint, model=args.model, api_key_env=args.api_key_env, agent=args.agent, token=token, only=args.only, server=args.llama_server, model_url=args.model_url, server_args=args.server_arg, runtime=args.runtime, runtime_version=args.runtime_version, overrides={"kind": args.kind, "gpu": args.gpu, "vram": args.vram, "ram": args.ram}, dry_run=args.dry_run, size=args.size, progress=lambda line: print(line, file=sys.stderr, flush=True)) except BenchError as exc: print(json.dumps({"error": str(exc)}), file=sys.stderr) return 1 except KeyboardInterrupt: print(json.dumps({"error": "Stopped; nothing was posted"}), file=sys.stderr) return 130 print(json.dumps(result, indent=2)) # Nothing answered is a failure, not a clean run. A refusal is posted, but the model answered nothing when every answer is a refusal. return 0 if result.get("answered") or args.dry_run else 1 if __name__ == "__main__": sys.exit(main())