如何运行 Underdog Saluki 27B：2-bit Qwen3.8-27B 工具调用指南

在 Google 上添加为首选来源

在本教程中，我们使用 Underdog Saluki 27B，这是 Qwen3.8-27B 的一个 7.9 GB 构建版本，其开发者称它在约每权重两比特的情况下仍保持工具调用能力完整，甚至在他们自建的 120 项任务基准上略胜完整的 54 GB 模型。Saluki 基于 ISTA-DASLab 对同一模型的 GSQ-RCO 量化，因此我们将两个文件并排放入原生 llama.cpp 中。我们先从文件本身入手：对两个 GGUF 头做逐张量差异比对，然后进行字节级比较，以判断 Saluki 是 ISTA 比特的重新混合，还是一个被改动的模型。接着我们在单块 GPU 上分别服务每个文件，读取其内存占用和速度，并用 BFCL 自带的检查器（严格与宽松解析）在来自伯克利函数调用排行榜的相同带种子 120 项任务上为两者打分。复制代码已复制使用其他浏览器

import os import re import ast import sys import json import math import copy import enum import glob import time import types import random import shutil import struct import hashlib import tarfile import platform import traceback import subprocess import collections import urllib.request import urllib.error

RESULTS = {} # step -> one-line result, for the summary table CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run) CURRENT = ["1"]

def banner(title): print("\n" + "=" * 78) print(title) print("=" * 78)

def section(name): def wrap(fn): def run(*a, **kw): CURRENT[0] = name.split(".")[0] banner(name) t0 = time.time() try: out = fn(*a, **kw) RESULTS[name] = out if isinstance(out, str) else "ok" return out except Exception as e: RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}" print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}") traceback.print_exc(limit=3) return None finally: print(f"\n[{time.time() - t0:.1f} s]") json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1) return run return wrap

def claim(text, held): CLAIMS.append((CURRENT[0], text, held)) return held

banner("1. Platform, pins and helpers") SYSTEM, MACHINE = platform.system(), platform.machine() ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac" WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work")) for d in ("models", "llama", "bfcl", "runs"): os.makedirs(os.path.join(WORK, d), exist_ok=True)

# 笔记本下载的所有内容都按大小和 SHA-256 固定。 LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2 LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844, "61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"), "Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509, "19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM] MODELS = { "Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c", file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152, sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"), "ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440", file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472, sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"), } # BFCL v4 data and BFCL's own checker code, at one gorilla commit. GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/" "berkeley-function-call-leaderboard/bfcl_eval/") BFCL_FILES = { "data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991", "data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a", "data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6", "data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84", "data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579", "data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc", "data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047", "data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0", "data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3", "eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538", "constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a", "constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd", "utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd", "model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3", } # Underdog Bench 的五类任务，每类 24 项；我们自行抽取带种子的样本，因为其任务列表并未公开。 KINDS = [("simple_python", "调用正确的那个函数"), ("multiple", "从多个函数中选出正确的那个"), ("irrelevance", "知道何时没有工具适用"), ("parallel", "同一函数，多次调用"), ("parallel_multiple", "多个不同的函数")] N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080 # llama-server 自身的默认值是 temperature 0.8 和 top_k 40，因此每个请求都会声明自己的采样器。 GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)

def sh(cmd, **kw): return subprocess.run(cmd, capture_output=True, text=True, **kw)

def sha256_file(path, chunk=1 {ver}")

for rel, digest in BFCL_FILES.items(): dest = os.path.join(WORK, "bfcl", rel.replace("/", "__")) fetch(GORILLA + rel, dest, sha=digest) print(f"BFCL v4：来自 gorilla@6ea5797 的 {len(BFCL_FILES)} 个文件（5 个任务文件、4 个答案文件、5 个检查器源码），全部经过 SHA-256 校验")

for tag, m in MODELS.items(): path = os.path.join(WORK, "models", m["file"]) had = os.path.exists(path) and os.path.getsize(path) == m["size"] url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}" dl = fetch(url, path, m["size"]) # 现在检查大小，下面检查哈希 t0 = time.time() ok = sha256_file(path) == m["sha256"] how = "已在磁盘上" if had else f"下载耗时 {dl:.0f} 秒（{m['size'] / dl / 1e6:.0f} MB/s）" print(f"{tag:7s}: {m['file']} {m['size']:,} 字节，{how}；SHA-256 {'匹配' if ok else '不匹配'} " f"Hub 的 LFS 固定值（计算哈希耗时 {time.time() - t0:.0f} 秒）") claim(f"{m['file']} 与 {m['repo']}@{m['rev'][:7]} 处固定的文件逐字节一致", ok) assert ok return f"llama.cpp {LLAMA_TAG}；Saluki 7.90 GB 和 ISTA IQ2_XS 8.42 GB 已通过 SHA-256 验证"

download_step()

我们下载 llama.cpp b11565 的 CUDA 构建、两个 2-bit 文件以及 BFCL 数据，并逐字节对照其固定值进行校验。Hugging Face 的下载支持断点续传，哈希采用固定修订版本下 Hub 自身的 LFS 摘要，因此若文件被悄悄替换，笔记本会直接停止，而不会改变其结果。BFCL 的任务文件、答案文件和检查器代码均来自同一个 gorilla 提交。该单元格会打印下载耗时，这是 Colab 上整个运行中最慢的部分。复制代码已复制使用其他浏览器

GGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144), 16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50), 21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56), 30: ("BF16", 1, 2)} # 名称、每块的值数、每块的字节数（ggml-common.h）

def read_gguf(path): """GGUF v3 头：元数据、张量表（名称、维度、类型、偏移）以及张量数据的起始位置。""" f = open(path, "rb") u = lambda fmt: struct.unpack(" a["bytes"]; down += b["bytes"] 5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}") for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]): if r["n"] {b}" for (a, b), n in moves.most_common())) print(f" {down} 个获得更少字节，{up} 个获得更多") claim("每个类型发生变化的张量都获得更少的比特；没有任何张量获得更多", up == 0 and down == changed) untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6] print(f" 完全未发生变化的类别：{', '.join(sorted(untouched))}") proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c claim("每个注意力和 DeltaNet 投影矩阵都保持其类型（只有小的 alpha/beta 门发生变化）", all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"])))) keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t")) print(f"\n不同的元数据键：{keys}") print(f"Saluki 的头仍然写着：general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}") print(f"两个头中存储的采样参数：temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, " f"top_p {kv_s['general.sampling.top_p']:.2f}（这些是 Qwen 的思考模式设置；第 5 步展示服务器如何处理它们）") saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6 return f"{len(t_s)} 个张量中有 {changed} 个类型发生变化，全部为降级；小了 {saved:.0f} MB"

header_diff()

用纯 Python 读取两个 GGUF 头，可以准确看出 Saluki 改动了什么。两个文件都有相同的 851 个张量，形状也相同，类型表能预测每个文件的字节大小。Saluki 改变了 146 个张量的类型，且每一处改动都削减了比特：大多数落在前馈下投影和输出层上，小的 DeltaNet 门向量从 BF16 降到 Q8_0。没有任何注意力或 DeltaNet 投影矩阵发生变化，头中仍将 ISTA DASLab 标为量化者。复制代码已复制使用其他浏览器

@section("4. 字节级溯源：未变化的张量是否就是 ISTA 自己的字节？") def provenance(): import numpy as np (_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"] ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r") mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r") q = dict(k=0, same=0, eq_bytes=0, bytes=0) f32 = collections.defaultdict(lambda: [0, 0, []]) # 类型 -> [相同, 不同, 相对 L2 变化] for a, b in zip(t_i, t_s): if a["type"] != b["type"]: continue x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]] y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]] if a["type"] == "F32": kind = re.sub(r"blk\.\d+\.", "", a["name"]) u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64) same = bool(np.array_equal(u, v)) f32[kind][0 if same else 1] += 1 if not same: f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u)) else: eq = int(np.count_nonzero(x == y)) q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"] print(f"保持原类型的量化矩阵：{q['same']} / {q['k']} 字节完全相同，但其 {q['bytes'] / 1e9:.2f} GB 字节中有 " f"{100 * q['eq_bytes'] / q['bytes']:.1f}% 相等") print(f"\n未量化的 F32 向量（从未重新编码，因此任何变化都意味着权重发生了变化）：") print(f" {'类型':28s} {'相同':>9s} {'变化':>8s} 相对变化中位数") for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]): med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-" print(f" {kind:28s} {same:9d} {diff:8d} {med}") changed = sum(v[1] for v in f32.values()) total = sum(v[0] + v[1] for v in f32.values()) claim("Saluki 中类型未变的张量与 ISTA 的逐字节相同（纯粹是类型重混）", q["same"] == q["k"] and changed == 0) claim("Saluki 的 F32 范数和门向量与 ISTA 的不同，因此其权重被改动过，而不只是重新编码", changed > 0) return (f"{q['same']}/{q['k']} 个同类型矩阵字节完全相同，其字节中有 {100 * q['eq_bytes'] / q['bytes']:.1f}% 相等；" f"{changed}/{total} 个 F32 向量发生变化")

provenance()

类型相同并不意味着权重相同，因此我们比较了 705 个保持原类型的张量的字节。352 个量化矩阵中只有一个字节完全相同，但它们的字节有 98.4% 匹配，这与对底层权重做小幅修改后在同一网格上重新量化的情况一致。未量化的范数和门向量给出了定论：大多数变化的中位数只有万分之几。Saluki 是一个经过轻微改动的模型，而不是对 ISTA 文件的重新编码。复制代码已复制使用其他浏览器

SERVER = {}

def start_server(tag): """每个模型一个 llama-server，一个槽位，所有层都在 GPU 上，保留调试日志以便处理未解析的回复。""" stop_server() log_path = os.path.join(WORK, "runs", f"server_{tag}.log") cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT), "-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"] t0 = time.time() proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV) while True: try: if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200: break except urllib.error.HTTPError as e: assert e.code == 503, e except urllib.error.URLError: pass assert proc.poll() is None, f"llama-server 已退出；请查看 {log_path}" assert time.time() - t0 {'取自 GGUF 头' if from_file else 'llama.cpp 内置值，而非头部'}；模型卡建议思考时用 0.6，" f" 因此下面每个请求都声明自己的采样器") claim("llama-server 从 GGUF 的 general.sampling 键获取默认采样器", from_file) caps = props.get("chat_template_caps", {}) print("模板支持：" + "、".join(k.replace("supports_", "") for k, v in caps.items() if v)) msgs = [{"role": "user", "content": "奥斯陆的天气怎么样？"}] tool = [{"type": "function", "function": {"name": "get_weather", "description": "获取某个城市的当前天气", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}] for label, kw in (("关闭思考", {"enable_thinking": False}), ("开启思考，低强度", {"reasoning_effort": "low"}), ("开启思考，默认", {})): prompt = api("/apply-template", {"messages": msgs, "tools": tool, "chat_template_kwargs": kw})["prompt"] n = len(api("/tokenize", {"content": prompt, "add_special": False})["tokens"]) print(f" {label:24s}: {n:3d} 个提示词 token，结尾为 {prompt[-28:]!r}") return f"KV {kv_tok // 1024} KiB/token，状态 {rs / 2**20:.0f} MiB；服务器缓冲区 {sum(mem[k] for k in ('weights', 'KV cache', 'recurrent state', 'compute')) / 1024:.1f} GiB"

SERVER_MEM = {} architecture_and_server()

Qwen3.8-27B 是一个混合模型：其 64 个块中有 48 个是带固定循环状态的 Gated DeltaNet 线性注意力层，只有每第四个块保留完整注意力。因此 KV 缓存每个 token 花费 64 KiB 而不是 256 KiB，另外每个序列约有 150 MiB 的状态。我们以单槽位启动 llama-server，从日志中读取其缓冲区，检查其默认采样器来自何处，并查看聊天模板在关闭思考、开启思考以及低强度下构建的提示词。复制代码已复制使用其他浏览器

SPEED = {}

def speed_probe(tag): """根据服务器自身的计时得出预填充和解码速率：两种提示长度，强制输出 128 个 token。""" rng = random.Random(SEED) words = "def class return import yield lambda tensor gradient matrix kernel cache token buffer layer".split() rows = [] for target in (256, 1024): text = " ".join(rng.choice(words) for _ in range(target)) ids = api("/tokenize", {"content": text, "add_special": False})["tokens"][:target] r = api("/completion", {"prompt": ids, "n_predict": 128, "ignore_eos": True, "cache_prompt": False, **GREEDY}) t = r["timings"] rows.append((t["prompt_n"], t["prompt_per_second"], t["predicted_n"], t["predicted_per_second"])) print(f" {tag:7s} prompt {t['prompt_n']:5d} tok at {t['prompt_per_second']:7.1f} tok/s | " f"decode {t['predicted_n']} tok at {t['predicted_per_second']:5.2f} tok/s") SPEED[tag] = rows return rows

@section("6. 由服务器测量的预填充和解码速度") def speed_step(): print(f"hardware: {HW}") rows = speed_probe("Saluki") return f"Saluki on {HW}: prefill {rows[1][1]:.0f} tok/s at {rows[1][0]} tokens, decode {rows[1][3]:.1f} tok/s"

speed_step()

速度来自服务器自身的计时，而不是围绕请求掐秒表。我们发送一个 256 token 和一个 1,024 token 的提示，关闭提示缓存并强制输出 128 个 token，从而得到每种长度下的预填充速率和解码速率。一个稠密的 27B 模型每生成一个 token 都要读取其完整的权重文件，因此解码速度主要取决于内存带宽；同样的探测稍后也会在 ISTA 的文件上再跑一遍，以便进行同类对比。复制代码已复制使用其他浏览器

def load_bfcl_checker(): """BFCL 自带的检查器函数，从固定版本的源码中执行，其包导入已被替换。""" src = lambda rel: open(os.path.join(WORK, "bfcl", rel.replace("/", "__"))).read()

def run(rel, ns, only=None): tree = ast.parse(src(rel)) body = [n for n in tree.body if not isinstance(n, (ast.Import, ast.ImportFrom))] if only: body = [n for n in body if getattr(n, "name", None) in only] exec(compile(ast.Module(body=body, type_ignores=[]), rel, "exec"), ns) return ns enums = run("constants/enums.py", {"Enum": enum.Enum}) tmap = run("constants/type_mappings.py", {}) model_cfg = collections.defaultdict(lambda: types.SimpleNamespace(underscore_to_dot=True)) # OpenAI 风格的 FC 模型 chk = run("eval_checker/ast_eval/ast_checker.py", {"re": re, "Language": enums["Language"], "MODEL_CONFIG_MAPPING": model_cfg, "JAVA_TYPE_CONVERSION": tmap["JAVA_TYPE_CONVERSION"], "JS_TYPE_CONVERSION": tmap["JS_TYPE_CONVERSION"], "java_type_converter": None, "js_type_converter": None}) utl = run("utils.py", {}, {"is_function_calling_format_output", "is_empty_output"}) conv = run("model_handler/utils.py", {"copy": copy, "re": re, "json": json, "ModelStyle": enums["ModelStyle"], "GORILLA_TO_OPENAPI": tmap["GORILLA_TO_OPENAPI"]}, {"_cast_to_openai_type", "convert_to_tool"}) return types.SimpleNamespace( ast_checker=lambda funcs, out, gt, cat: chk["ast_checker"](funcs, out, gt, enums["Language"].PYTHON, cat, "fc-model"), is_empty=utl["is_empty_output"], to_tools=lambda funcs: conv["convert_to_tool"](funcs, tmap["GORILLA_TO_OPENAPI"], enums["ModelStyle"].OPENAI_COMPLETIONS))

def load_tasks(): tasks = [] for kind, label in KINDS: rows = [json.loads(l) for l in open(os.path.join(WORK, "bfcl", f"data__BFCL_v4_{kind}.json")) if l.strip()] gt = {} if kind != "irrelevance": for l in open(os.path.join(WORK, "bfcl", f"data__possible_answer__BFCL_v4_{kind}.json")): if l.strip(): d = json.loads(l); gt[d["id"]] = d["ground_truth"] for r in sorted(random.Random(f"{SEED}-{kind}").sample(rows, N_PER_KIND), key=lambda r: int(r["id"].rsplit("_", 1)[1])): tasks.append(dict(id=r["id"], kind=kind, messages=r["question"][0], functions=r["function"], gt=gt.get(r["id"]))) return tasks

TOOL_BLOCK = re.compile(r" (.*?)(?: |$)", re.S) FUNC_XML = re.compile(r" \s]+)\s*>(.*?)(?: |$)", re.S) PARAM_XML = re.compile(r" \s]+)\s*>\n?(.*?)\n? ", re.S)

def lenient_calls(raw, functions): """对模型原始文本的宽容读取：任何块，无论是否闭合，在内部或外部。值按函数自身的 schema（BFCL 类型）进行类型化。不属于 BFCL；仅用于“宽松”列。""" raw = raw.split(" ")[-1] # 开启思考时，推理内容在前 types_ = {re.sub(r"\.", "_", f["name"]): {k: v.get("type", "string") for k, v in f["parameters"]["properties"].items()} for f in functions} # 工具名称发送时已将点转为下划线 calls = [] for fname, body in FUNC_XML.findall(raw): fname = fname.strip().strip('"') args = {} for p, v in PARAM_XML.findall(body): t = types_.get(re.sub(r"\.", "_", fname), {}).get(p, "string") v = v.strip() if t != "string" else v try: args[p] = (int(v) if t == "integer" else float(v) if t in ("float", "number") else v.lower() == "true" if t == "boolean" and v.lower() in ("true", "false") else json.loads(v) if t in ("array", "tuple", "dict", "object") else v) except ValueError: try: args[p] = ast.literal_eval(v) except Exception: args[p] = v calls.append({fname: args}) return calls

def score(task, calls): """BFCL 的规则：无关类在无调用时通过；其他类型则经过 BFCL 的 ast_checker。""" if task["kind"] == "irrelevance": return CHECK.is_empty(calls), "made a call" if calls else None if not calls: return False, "no call" try: r = CHECK.ast_checker(task["functions"], calls, task["gt"], task["kind"]) except Exception as e: # 检查器无法遍历的畸形参数 return False, f"checker_error:{type(e).__name__}" return bool(r["valid"]), r.get("error_type")

def run_task(task, think=False, max_tokens=512): body = {"messages": task["messages"], "tools": CHECK.to_tools(task["functions"]), "max_tokens": max_tokens, "cache_prompt": False, "verbose": True, **GREEDY, "chat_template_kwargs": {"reasoning_effort": "low"} if think else {"enable_thinking": False}} log_pos = os.path.getsize(SERVER["log"]) t0 = time.time() rec = dict(id=task["id"], kind=task["kind"], think=think) try: r = api("/v1/chat/completions", body) msg = r["choices"][0]["message"] rec.update(status=200, finish=r["choices"][0]["finish_reason"], raw=r["__verbose"]["content"], prompt_n=r["timings"]["prompt_n"], out_n=r["timings"]["predicted_n"], reasoning_chars=len(msg.get("reasoning_content") or "")) rec["calls"] = [{tc["function"]["name"]: json.loads(tc["function"]["arguments"])} for tc in msg.get("tool_calls") or []] except urllib.error.HTTPError as e: # 当解析器无法读取回复时，原版 llama.cpp 会返回 500 time.sleep(0.5) with open(SERVER["log"], errors="replace") as f: f.seek(log_pos) new = f.read() m = re.findall(r"=== BEGIN ===\n(.*?)\n=== END ===", new, re.S) rec.update(status=e.code, finish="parse error", raw=m[-1] if m else "", calls=None, prompt_n=None, out_n=None) rec["seconds"] = time.time() - t0 rec["strict"], rec["strict_why"] = score(task, rec["calls"]) if rec["calls"] is not None else (False, "unparsed (HTTP 500)") if rec["strict"]: rec["lenient"] = True else: rec["lenient"], _ = score(task, lenient_calls(rec["raw"], task["functions"])) return rec

def bench(tag, tasks): out, t0 = [], time.time() for i, task in enumerate(tasks): out.append(run_task(task)) if (i + 1) % 30 == 0: print(f" {tag}: {i + 1}/{len(tasks)} done, {time.time() - t0:.0f} s") json.dump(out, open(os.path.join(WORK, "runs", f"bfcl_{tag}.json"), "w"), indent=1) return out

def ci(k, n, alpha=0.05): """Clopper-Pearson 区间，通过对二项尾部进行二分法求解。""" def cdf(x, p): return sum(math.comb(n, i) * p ** i * (1 - p) ** (n - i) for i in range(x + 1))

def solve(f, target, rising): lo, hi = 0.0, 1.0 for _ in range(60): mid = (lo + hi) / 2 lo, hi = (mid, hi) if (f(mid) 8s} {'lenient':>8s} replies: structured / plain text / unparsed (500) / cut off") for kind, label in KINDS: rs = [r for r in recs if r["kind"] == kind] st = collections.Counter("unparsed" if r["status"] != 200 else "cut off" if r["finish"] == "length" else "structured" if r["calls"] else "plain text" for r in rs) print(f" {label:44s} {sum(r['strict'] for r in rs):3d}/{len(rs): 7s} {'ISTA':>6s} | {'lenient':>7s} {'':>5s} | vendor's Saluki (of 24)") for kind, label in KINDS: s = [r for r in S if r["kind"] == kind]; i = [r for r in I if r["kind"] == kind] print(f"{label:38s} {sum(r['strict'] for r in s):7d} {sum(r['strict'] for r in i):6d} | " f"{sum(r['lenient'] for r in s):7d} {sum(r['lenient'] for r in i):5d} | {vendor[kind]:3d}") out = {} for mode in ("strict", "lenient"): a, b = [r[mode] for r in S], [r[mode] for r in I] b10, b01, p = mcnemar(a, b) out[mode] = (sum(a), sum(b), b10, b01, p) print(f"{mode:8s}: Saluki {sum(a)}/120, ISTA {sum(b)}/120; Saluki-only passes {b10}, ISTA-only {b01}, exact McNemar p = {p:.3f}") print("vendor's Underdog Bench (different 120 tasks, unpublished): Saluki 88, ISTA IQ2_XS 76, full Qwen3.8-27B 84") sa, ia, b10, b01, p = out["strict"] claim("Saluki passes more of our 120 tasks than its parent (strict scoring)", sa > ia) claim("the difference is significant at 5% (exact McNemar)", p = pick_i) return f"strict: Saluki {sa}/120 vs ISTA {ia}/120 (McNemar p = {p:.2f}); lenient: {out['lenient'][0]} vs {out['lenient'][1]}"

head_to_head()

由于两个模型回答的是同一批任务，因此比较是配对的。该单元格会打印按类型划分的分数，并与厂商给出的 Saluki 各类型数字并列，同时对两个文件结果不一致的任务运行精确 McNemar 检验，分别针对严格和宽松评分。120 项任务中相差几项属于运行间噪声，厂商页面本身也承认这一点，因此该检验告诉我们样本是否显示出真实差异。复制代码已复制使用其他浏览器

@section("11. 失败从何而来") def failures(): for tag in ("Saluki", "ISTA"): recs = RUNS[tag] why = collections.Counter((r["strict_why"] or "?").split(":")[-1] for r in recs if not r["strict"]) print(f"{tag}: {sum(not r['strict'] for r in recs)} failures -> " + ", ".join(f"{k} {v}" for k, v in why.most_common())) rescued = [r for r in recs if r["lenient"] and not r["strict"]] print(f" lenient reading rescues {len(rescued)}: " + ", ".join(r["id"] for r in rescued[:8])) ex = next((r for r in RUNS["Saluki"] if not r["strict"] and r["kind"] != "irrelevance"), None) if ex: task = next(t for t in TASKS if t["id"] == ex["id"]) print(f"\nexample, {ex['id']} ({ex['strict_why']}):") print(" expected:", json.dumps(task["gt"])[:300]) print(" model :", (json.dumps(ex["calls"]) if ex["calls"] is not None else ex["raw"])[:300]) irr = [r for r in RUNS["Saluki"] if r["kind"] == "irrelevance" and not r["strict"]] if irr: print(f"\nirrelevance: Saluki made a call on {len(irr)} of {N_PER_KIND} prompts that no tool fits, e.g. {irr[0]['id']}: " f"{json.dumps(irr[0]['calls'])[:160]}") un = sum(r["status"] != 200 for tag in RUNS for r in RUNS[tag]) return f"{un} replies across both models were unparseable by the stock server"

failures()

分数掩盖了调用出错的位置，因此本节按 BFCL 自身的错误类型拆解失败情况：调用次数错误、参数错误或缺失、在没有合适工具时发起调用，以及原版服务器无法解析的回复。它还列出了宽松解析所挽救的任务，并将一个失败任务与其预期答案并排打印，这是区分模型错误与评分假象的最快方式。复制代码已复制使用其他浏览器

banner("SUMMARY") for name, res in RESULTS.items(): print(f" {name:<100s} {res}") print("\n本次运行中判定的主张：") for step, text, held in CLAIMS: print(f" [{'成立' if held else '不成立' if held is False else '未定'}] 步骤 {step}: {text}") failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")] print(f"\n{sum(h is True for *_, h in CLAIMS)} / {len(CLAIMS)} 项主张成立；失败或跳过的步骤：{failed or '无'}") print(f"运行环境：{HW}；llama.cpp {LLAMA_TAG}；Python {platform.python_version()}") print("此处未测试：完整规模的 Qwen3.8-27B；Underdog Bench 自有的 120 项任务及 100 项并行集（未公开）；" "SWE-bench、WebWalkerQA、IFEval 和 AIME；视觉附加组件；Mia 和 Ternary Bonsai 2；默认努力程度下的思考模式。")

摘要会打印每个步骤的一行结果和主张台账（成立、不成立或未定），以及硬件和 llama.cpp 版本。最后列出该 notebook 未测试的内容：完整规模的 Qwen3.8-27B、厂商自有的任务列表、智能体与数学基准，以及视觉附加组件。

总之，Saluki 并不完全如其文件所示。它的文件头保留了 ISTA 的 851 张量布局，并且只在 ISTA 花费更多比特的地方节省比特，然而其字节和未量化向量显示这是一个权重被改动过的模型，这与 Underdog 称其为预览检查点的说法一致。这使得与其父模型的工具调用对比变成了对该训练的检验，而不仅仅是量化。两个文件都在原版 llama.cpp 中逐个运行，BFCL 自带的检查器在相同任务上给出配对比较。小样本、贪心解码和单次运行限制了任何单个 notebook 所能断言的内容，因此置信区间与分数同样重要。

Sana Hassan+ 发布简介 ⮌

Sana Hassan 是 Marktechpost 的咨询实习生，也是 IIT Madras 的双学位学生，热衷于将技术和 AI 应用于解决现实世界的挑战。他对解决实际问题有浓厚兴趣，为 AI 与现实生活解决方案的交汇带来了全新视角。

Sana Hassan《Cactus Whistle 编码指南：一个 16.9 MB 的语音模型能否媲美 Whisper Base？在 CPU 上的词错误率、速度、流式处理、时间戳、关键词以及语音到工具调用

$uv pip install omni-parse-bench✓文本✓表格✓布局→ 按页评分在 GitHub 上加星→赞助

如何运行 Underdog Saluki 27B：2-bit Qwen3.8-27B 工具调用指南

主题

标签

正文 · AI 翻译