Files
Pizza 11b49d3933 feat: milestone 3 complete + QA benchmark
Web UI (tinyclaw web):
- Flask + SSE real-time progress streaming
- Queue view: add/delete tasks, status colours, error display
- Live log panel with coloured output
- History tab with unified diffs (Rosé Pine theme)
- Model picker dropdown
- Auto-refresh every 5s

History (history.py):
- JSONL-based, records before/after content + unified diff
- Wired into driver for EDIT and CREATE actions
- Failed tasks also recorded

QA benchmark (tinyclaw qa <models...>):
- 5 fixed tasks: divide/greet/is_even/repeat/safe_get
- Scores syntax validity + instruction following
- Comparison table: pass%, syntax%, latency, cost

QA results (3 models):
- gemma-3-12b-it:  80% pass, 100% syntax, ~944ms  ← winner
- gemma-3-4b-it:   60% pass, 100% syntax, ~917ms
- qwen2.5-coder-7b: 0% pass,   0% syntax (max_tokens issue — hits 512 limit mid-function)
2026-04-03 01:27:41 +00:00

249 lines
7.4 KiB
Python

"""TinyClaw model QA — benchmark small models against a fixed task suite."""
import tempfile, time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Callable
from tinyclaw.client import generate
from tinyclaw.extractor import extract_function, splice_function
from tinyclaw.prompt import build_edit_prompt
from tinyclaw.verifier import verify_syntax_string
# ── benchmark tasks ───────────────────────────────────────────────────────────
BENCHMARK_SOURCE = '''\
def divide(a, b):
return a / b
def greet(name):
return "hello " + name
def is_even(n):
return n % 2 == 1
def repeat(s, n):
result = ""
for i in range(n):
result = result + s
return result
def safe_get(d, key):
return d[key]
'''
@dataclass
class BenchTask:
func: str
instruction: str
# returns (passed, reason) given the new function source
check: Callable[[str], tuple[bool, str]]
def _contains(keywords: list[str]):
def _check(src: str) -> tuple[bool, str]:
src_lower = src.lower()
for kw in keywords:
if kw.lower() not in src_lower:
return False, f"missing '{kw}'"
return True, "ok"
return _check
BENCHMARK_TASKS = [
BenchTask(
func="divide",
instruction="add ZeroDivisionError guard, raise ValueError with message",
check=_contains(["valueerror", "== 0"]),
),
BenchTask(
func="greet",
instruction="capitalize name and add exclamation mark",
check=_contains(["capitalize()", "!"]),
),
BenchTask(
func="is_even",
instruction="fix the bug — function should return True for even numbers",
check=lambda src: (
("% 2 == 0" in src or "% 2==0" in src, "expected '% 2 == 0'")
if ("% 2 == 0" in src or "% 2==0" in src)
else (False, "still has wrong logic")
),
),
BenchTask(
func="repeat",
instruction="rewrite using string multiplication instead of a loop",
check=_contains(["* n", "return s"]),
),
BenchTask(
func="safe_get",
instruction="add KeyError handling, return None if key missing",
check=_contains(["keyerror", "none"]),
),
]
@dataclass
class TaskResult:
func: str
instruction: str
passed: bool
syntax_ok: bool
reason: str
prompt_tokens: int = 0
completion_tokens: int = 0
cost: float = 0.0
latency_ms: int = 0
output_preview: str = ""
error: str = ""
@dataclass
class ModelResult:
model: str
tasks: list[TaskResult] = field(default_factory=list)
@property
def pass_rate(self) -> float:
if not self.tasks: return 0.0
return sum(1 for t in self.tasks if t.passed) / len(self.tasks)
@property
def syntax_rate(self) -> float:
if not self.tasks: return 0.0
return sum(1 for t in self.tasks if t.syntax_ok) / len(self.tasks)
@property
def avg_latency_ms(self) -> float:
times = [t.latency_ms for t in self.tasks if t.latency_ms]
return sum(times) / len(times) if times else 0.0
@property
def total_cost(self) -> float:
return sum(t.cost for t in self.tasks)
def run_qa(
models: list[str],
api_key: str | None = None,
api_base: str | None = None,
verbose: bool = False,
) -> list[ModelResult]:
"""Run all benchmark tasks against each model. Returns results."""
from tinyclaw.driver import _load_dotenv
_load_dotenv()
results = []
for model in models:
print(f"\n{'='*60}")
print(f"Model: {model}")
print('='*60)
mr = _bench_model(model, api_key=api_key, api_base=api_base, verbose=verbose)
results.append(mr)
_print_model_summary(mr)
print(f"\n{'='*60}")
print("SUMMARY")
print('='*60)
_print_comparison(results)
return results
def _bench_model(
model: str,
api_key: str | None = None,
api_base: str | None = None,
verbose: bool = False,
) -> ModelResult:
mr = ModelResult(model=model)
with tempfile.TemporaryDirectory() as tmpdir:
target = Path(tmpdir) / "bench.py"
target.write_text(BENCHMARK_SOURCE)
for task in BENCHMARK_TASKS:
tr = _run_task(task, str(target), model, api_key, api_base, verbose)
mr.tasks.append(tr)
icon = "✓" if tr.passed else ("~" if tr.syntax_ok else "✗")
print(f" {icon} {task.func:<12} {tr.reason:<35} {tr.latency_ms:>5}ms {tr.prompt_tokens}pt/{tr.completion_tokens}ct")
# Reset file between tasks
target.write_text(BENCHMARK_SOURCE)
return mr
def _run_task(
task: BenchTask,
filepath: str,
model: str,
api_key: str | None,
api_base: str | None,
verbose: bool,
) -> TaskResult:
t0 = time.monotonic()
try:
info = extract_function(filepath, task.func)
system, user = build_edit_prompt(info.source, task.instruction)
resp = generate(
prompt=user, model=model, system=system,
api_key=api_key, api_base=api_base,
max_tokens=512, temperature=0.1,
)
latency_ms = int((time.monotonic() - t0) * 1000)
# strip fences
raw = resp.content.strip()
if raw.startswith("```"):
lines = raw.splitlines()
end = len(lines) - 1 if lines[-1].strip() == "```" else len(lines)
raw = "\n".join(lines[1:end]).strip()
vr = verify_syntax_string(raw, label=f"{task.func}")
if not vr.ok:
return TaskResult(
func=task.func, instruction=task.instruction,
passed=False, syntax_ok=False,
reason=f"syntax: {vr.error[:60]}",
prompt_tokens=resp.prompt_tokens,
completion_tokens=resp.completion_tokens,
cost=resp.cost or 0.0, latency_ms=latency_ms,
output_preview=raw[:120],
)
passed, reason = task.check(raw)
if verbose:
print(f"\n --- {task.func} output ---\n{raw}\n ---")
return TaskResult(
func=task.func, instruction=task.instruction,
passed=passed, syntax_ok=True, reason=reason,
prompt_tokens=resp.prompt_tokens,
completion_tokens=resp.completion_tokens,
cost=resp.cost or 0.0, latency_ms=latency_ms,
output_preview=raw[:120],
)
except Exception as e:
latency_ms = int((time.monotonic() - t0) * 1000)
return TaskResult(
func=task.func, instruction=task.instruction,
passed=False, syntax_ok=False,
reason=f"error: {str(e)[:60]}",
latency_ms=latency_ms, error=str(e),
)
def _print_model_summary(mr: ModelResult) -> None:
print(f"\n Pass: {mr.pass_rate:.0%} Syntax: {mr.syntax_rate:.0%} "
f"Avg latency: {mr.avg_latency_ms:.0f}ms Cost: ${mr.total_cost:.4f}")
def _print_comparison(results: list[ModelResult]) -> None:
header = f"{'Model':<45} {'Pass':>5} {'Syntax':>7} {'Latency':>9} {'Cost':>8}"
print(header)
print("-" * len(header))
for mr in sorted(results, key=lambda r: -r.pass_rate):
print(f"{mr.model:<45} {mr.pass_rate:>4.0%} {mr.syntax_rate:>5.0%} "
f"{mr.avg_latency_ms:>7.0f}ms ${mr.total_cost:>6.4f}")