Web UI (tinyclaw web): - Flask + SSE real-time progress streaming - Queue view: add/delete tasks, status colours, error display - Live log panel with coloured output - History tab with unified diffs (Rosé Pine theme) - Model picker dropdown - Auto-refresh every 5s History (history.py): - JSONL-based, records before/after content + unified diff - Wired into driver for EDIT and CREATE actions - Failed tasks also recorded QA benchmark (tinyclaw qa <models...>): - 5 fixed tasks: divide/greet/is_even/repeat/safe_get - Scores syntax validity + instruction following - Comparison table: pass%, syntax%, latency, cost QA results (3 models): - gemma-3-12b-it: 80% pass, 100% syntax, ~944ms ← winner - gemma-3-4b-it: 60% pass, 100% syntax, ~917ms - qwen2.5-coder-7b: 0% pass, 0% syntax (max_tokens issue — hits 512 limit mid-function)
249 lines
7.4 KiB
Python
249 lines
7.4 KiB
Python
"""TinyClaw model QA — benchmark small models against a fixed task suite."""
|
|
|
|
import tempfile, time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Callable
|
|
|
|
from tinyclaw.client import generate
|
|
from tinyclaw.extractor import extract_function, splice_function
|
|
from tinyclaw.prompt import build_edit_prompt
|
|
from tinyclaw.verifier import verify_syntax_string
|
|
|
|
# ── benchmark tasks ───────────────────────────────────────────────────────────
|
|
BENCHMARK_SOURCE = '''\
|
|
def divide(a, b):
|
|
return a / b
|
|
|
|
def greet(name):
|
|
return "hello " + name
|
|
|
|
def is_even(n):
|
|
return n % 2 == 1
|
|
|
|
def repeat(s, n):
|
|
result = ""
|
|
for i in range(n):
|
|
result = result + s
|
|
return result
|
|
|
|
def safe_get(d, key):
|
|
return d[key]
|
|
'''
|
|
|
|
@dataclass
|
|
class BenchTask:
|
|
func: str
|
|
instruction: str
|
|
# returns (passed, reason) given the new function source
|
|
check: Callable[[str], tuple[bool, str]]
|
|
|
|
|
|
def _contains(keywords: list[str]):
|
|
def _check(src: str) -> tuple[bool, str]:
|
|
src_lower = src.lower()
|
|
for kw in keywords:
|
|
if kw.lower() not in src_lower:
|
|
return False, f"missing '{kw}'"
|
|
return True, "ok"
|
|
return _check
|
|
|
|
|
|
BENCHMARK_TASKS = [
|
|
BenchTask(
|
|
func="divide",
|
|
instruction="add ZeroDivisionError guard, raise ValueError with message",
|
|
check=_contains(["valueerror", "== 0"]),
|
|
),
|
|
BenchTask(
|
|
func="greet",
|
|
instruction="capitalize name and add exclamation mark",
|
|
check=_contains(["capitalize()", "!"]),
|
|
),
|
|
BenchTask(
|
|
func="is_even",
|
|
instruction="fix the bug — function should return True for even numbers",
|
|
check=lambda src: (
|
|
("% 2 == 0" in src or "% 2==0" in src, "expected '% 2 == 0'")
|
|
if ("% 2 == 0" in src or "% 2==0" in src)
|
|
else (False, "still has wrong logic")
|
|
),
|
|
),
|
|
BenchTask(
|
|
func="repeat",
|
|
instruction="rewrite using string multiplication instead of a loop",
|
|
check=_contains(["* n", "return s"]),
|
|
),
|
|
BenchTask(
|
|
func="safe_get",
|
|
instruction="add KeyError handling, return None if key missing",
|
|
check=_contains(["keyerror", "none"]),
|
|
),
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class TaskResult:
|
|
func: str
|
|
instruction: str
|
|
passed: bool
|
|
syntax_ok: bool
|
|
reason: str
|
|
prompt_tokens: int = 0
|
|
completion_tokens: int = 0
|
|
cost: float = 0.0
|
|
latency_ms: int = 0
|
|
output_preview: str = ""
|
|
error: str = ""
|
|
|
|
|
|
@dataclass
|
|
class ModelResult:
|
|
model: str
|
|
tasks: list[TaskResult] = field(default_factory=list)
|
|
|
|
@property
|
|
def pass_rate(self) -> float:
|
|
if not self.tasks: return 0.0
|
|
return sum(1 for t in self.tasks if t.passed) / len(self.tasks)
|
|
|
|
@property
|
|
def syntax_rate(self) -> float:
|
|
if not self.tasks: return 0.0
|
|
return sum(1 for t in self.tasks if t.syntax_ok) / len(self.tasks)
|
|
|
|
@property
|
|
def avg_latency_ms(self) -> float:
|
|
times = [t.latency_ms for t in self.tasks if t.latency_ms]
|
|
return sum(times) / len(times) if times else 0.0
|
|
|
|
@property
|
|
def total_cost(self) -> float:
|
|
return sum(t.cost for t in self.tasks)
|
|
|
|
|
|
def run_qa(
|
|
models: list[str],
|
|
api_key: str | None = None,
|
|
api_base: str | None = None,
|
|
verbose: bool = False,
|
|
) -> list[ModelResult]:
|
|
"""Run all benchmark tasks against each model. Returns results."""
|
|
from tinyclaw.driver import _load_dotenv
|
|
_load_dotenv()
|
|
|
|
results = []
|
|
for model in models:
|
|
print(f"\n{'='*60}")
|
|
print(f"Model: {model}")
|
|
print('='*60)
|
|
mr = _bench_model(model, api_key=api_key, api_base=api_base, verbose=verbose)
|
|
results.append(mr)
|
|
_print_model_summary(mr)
|
|
|
|
print(f"\n{'='*60}")
|
|
print("SUMMARY")
|
|
print('='*60)
|
|
_print_comparison(results)
|
|
return results
|
|
|
|
|
|
def _bench_model(
|
|
model: str,
|
|
api_key: str | None = None,
|
|
api_base: str | None = None,
|
|
verbose: bool = False,
|
|
) -> ModelResult:
|
|
mr = ModelResult(model=model)
|
|
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
target = Path(tmpdir) / "bench.py"
|
|
target.write_text(BENCHMARK_SOURCE)
|
|
|
|
for task in BENCHMARK_TASKS:
|
|
tr = _run_task(task, str(target), model, api_key, api_base, verbose)
|
|
mr.tasks.append(tr)
|
|
icon = "✓" if tr.passed else ("~" if tr.syntax_ok else "✗")
|
|
print(f" {icon} {task.func:<12} {tr.reason:<35} {tr.latency_ms:>5}ms {tr.prompt_tokens}pt/{tr.completion_tokens}ct")
|
|
|
|
# Reset file between tasks
|
|
target.write_text(BENCHMARK_SOURCE)
|
|
|
|
return mr
|
|
|
|
|
|
def _run_task(
|
|
task: BenchTask,
|
|
filepath: str,
|
|
model: str,
|
|
api_key: str | None,
|
|
api_base: str | None,
|
|
verbose: bool,
|
|
) -> TaskResult:
|
|
t0 = time.monotonic()
|
|
try:
|
|
info = extract_function(filepath, task.func)
|
|
system, user = build_edit_prompt(info.source, task.instruction)
|
|
|
|
resp = generate(
|
|
prompt=user, model=model, system=system,
|
|
api_key=api_key, api_base=api_base,
|
|
max_tokens=512, temperature=0.1,
|
|
)
|
|
latency_ms = int((time.monotonic() - t0) * 1000)
|
|
|
|
# strip fences
|
|
raw = resp.content.strip()
|
|
if raw.startswith("```"):
|
|
lines = raw.splitlines()
|
|
end = len(lines) - 1 if lines[-1].strip() == "```" else len(lines)
|
|
raw = "\n".join(lines[1:end]).strip()
|
|
|
|
vr = verify_syntax_string(raw, label=f"{task.func}")
|
|
if not vr.ok:
|
|
return TaskResult(
|
|
func=task.func, instruction=task.instruction,
|
|
passed=False, syntax_ok=False,
|
|
reason=f"syntax: {vr.error[:60]}",
|
|
prompt_tokens=resp.prompt_tokens,
|
|
completion_tokens=resp.completion_tokens,
|
|
cost=resp.cost or 0.0, latency_ms=latency_ms,
|
|
output_preview=raw[:120],
|
|
)
|
|
|
|
passed, reason = task.check(raw)
|
|
if verbose:
|
|
print(f"\n --- {task.func} output ---\n{raw}\n ---")
|
|
|
|
return TaskResult(
|
|
func=task.func, instruction=task.instruction,
|
|
passed=passed, syntax_ok=True, reason=reason,
|
|
prompt_tokens=resp.prompt_tokens,
|
|
completion_tokens=resp.completion_tokens,
|
|
cost=resp.cost or 0.0, latency_ms=latency_ms,
|
|
output_preview=raw[:120],
|
|
)
|
|
|
|
except Exception as e:
|
|
latency_ms = int((time.monotonic() - t0) * 1000)
|
|
return TaskResult(
|
|
func=task.func, instruction=task.instruction,
|
|
passed=False, syntax_ok=False,
|
|
reason=f"error: {str(e)[:60]}",
|
|
latency_ms=latency_ms, error=str(e),
|
|
)
|
|
|
|
|
|
def _print_model_summary(mr: ModelResult) -> None:
|
|
print(f"\n Pass: {mr.pass_rate:.0%} Syntax: {mr.syntax_rate:.0%} "
|
|
f"Avg latency: {mr.avg_latency_ms:.0f}ms Cost: ${mr.total_cost:.4f}")
|
|
|
|
|
|
def _print_comparison(results: list[ModelResult]) -> None:
|
|
header = f"{'Model':<45} {'Pass':>5} {'Syntax':>7} {'Latency':>9} {'Cost':>8}"
|
|
print(header)
|
|
print("-" * len(header))
|
|
for mr in sorted(results, key=lambda r: -r.pass_rate):
|
|
print(f"{mr.model:<45} {mr.pass_rate:>4.0%} {mr.syntax_rate:>5.0%} "
|
|
f"{mr.avg_latency_ms:>7.0f}ms ${mr.total_cost:>6.4f}")
|