Files
Pizza 11b49d3933 feat: milestone 3 complete + QA benchmark
Web UI (tinyclaw web):
- Flask + SSE real-time progress streaming
- Queue view: add/delete tasks, status colours, error display
- Live log panel with coloured output
- History tab with unified diffs (Rosé Pine theme)
- Model picker dropdown
- Auto-refresh every 5s

History (history.py):
- JSONL-based, records before/after content + unified diff
- Wired into driver for EDIT and CREATE actions
- Failed tasks also recorded

QA benchmark (tinyclaw qa <models...>):
- 5 fixed tasks: divide/greet/is_even/repeat/safe_get
- Scores syntax validity + instruction following
- Comparison table: pass%, syntax%, latency, cost

QA results (3 models):
- gemma-3-12b-it:  80% pass, 100% syntax, ~944ms  ← winner
- gemma-3-4b-it:   60% pass, 100% syntax, ~917ms
- qwen2.5-coder-7b: 0% pass,   0% syntax (max_tokens issue — hits 512 limit mid-function)
2026-04-03 01:27:41 +00:00

364 lines
12 KiB
Python

"""Core driver loop: read queue → extract → prompt → call model → verify → commit."""
import os
import subprocess
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
def _load_dotenv(path: str = ".env") -> None:
"""Load KEY=VALUE pairs from .env into os.environ (no-op if missing)."""
p = Path(path)
if not p.exists():
return
for line in p.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, val = line.partition("=")
os.environ.setdefault(key.strip(), val.strip())
from tinyclaw import history as hist
from tinyclaw.client import generate
from tinyclaw.extractor import extract_function, splice_function, FunctionInfo
from tinyclaw.prompt import build_edit_prompt, build_create_prompt, estimate_tokens
from tinyclaw.queue import (
Task, TaskAction, TaskStatus,
parse_queue, write_queue, queue_summary,
)
from tinyclaw.verifier import verify_syntax_string, verify_function_present
MAX_RETRIES = 3
MAX_PROMPT_TOKENS = 600 # warn if exceeded
@dataclass
class RunConfig:
queue_path: str = "QUEUE.md"
model: str = "qwen/qwen3.5-1.5b"
api_base: str | None = None
api_key: str | None = None
dry_run: bool = False
verbose: bool = False
max_tokens: int = 1024
temperature: float = 0.1
auto_commit: bool = True
history_path: str = "HISTORY.jsonl"
@dataclass
class FailureEntry:
task: str
attempt: int
error: str
timestamp: str
@dataclass
class RunResult:
done: int = 0
failed: int = 0
skipped: int = 0
total_cost: float = 0.0
total_prompt_tokens: int = 0
total_completion_tokens: int = 0
failures: list[FailureEntry] = field(default_factory=list)
def run(config: RunConfig) -> RunResult:
"""Process the task queue. Returns a summary of results."""
_load_dotenv()
queue_path = config.queue_path
if not Path(queue_path).exists():
raise FileNotFoundError(f"Queue file not found: {queue_path}")
tasks = parse_queue(queue_path)
pending = [t for t in tasks if t.status == TaskStatus.PENDING]
_log(config, f"Queue: {queue_summary(tasks)}")
_log(config, f"Model: {config.model}")
if config.dry_run:
_log(config, "[dry-run] Tasks that would run:")
for t in pending:
_log(config, f" {t.action.value} {t.filepath}::{t.target} — {t.instruction}")
return RunResult()
result = RunResult()
for task in pending:
_log(config, f"\n→ {task.action.value} {task.filepath}::{task.target}")
_log(config, f" {task.instruction}")
success = False
for attempt in range(1, MAX_RETRIES + 1):
task.attempts = attempt
task.status = TaskStatus.RUNNING
try:
resp = _execute_task(task, config)
result.total_cost += resp.cost or 0.0
result.total_prompt_tokens += resp.prompt_tokens
result.total_completion_tokens += resp.completion_tokens
task.status = TaskStatus.DONE
success = True
_log(config, f" ✓ attempt {attempt} — {resp.prompt_tokens}pt / {resp.completion_tokens}ct")
break
except Exception as e:
task.error = str(e)
_log(config, f" ✗ attempt {attempt}: {e}")
result.failures.append(FailureEntry(
task=f"{task.action.value} {task.filepath}::{task.target}",
attempt=attempt,
error=str(e),
timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
))
if attempt < MAX_RETRIES:
import time
# Exponential backoff: 2s, 4s — longer on rate limit
wait = (2 ** attempt) * (5 if "429" in str(e) else 1)
_log(config, f" retrying in {wait}s...")
time.sleep(wait)
if not success:
if task.attempts >= MAX_RETRIES:
task.status = TaskStatus.SKIPPED
_log(config, f" → skipped after {MAX_RETRIES} attempts")
result.skipped += 1
else:
task.status = TaskStatus.FAILED
result.failed += 1
if config.history_path:
hist.record(
config.history_path,
action=task.action.value, filepath=task.filepath, target=task.target,
instruction=task.instruction, status=task.status.value,
error=task.error,
)
else:
result.done += 1
# Persist queue state after every task
write_queue(queue_path, tasks)
# Auto-commit on success
if success and config.auto_commit:
_git_commit(task, config)
_log(config, f"\nDone: {result.done} ✓ Failed: {result.failed} ✗ Skipped: {result.skipped} →")
if result.total_cost:
_log(config, f"Cost: ${result.total_cost:.4f} Tokens: {result.total_prompt_tokens}p / {result.total_completion_tokens}c")
if result.failures:
_write_failure_log(result.failures, config)
return result
def _write_failure_log(failures: list[FailureEntry], config: RunConfig) -> None:
"""Append failures to FAILURES.md next to the queue file."""
log_path = Path(config.queue_path).parent / "FAILURES.md"
lines = []
if not log_path.exists():
lines.append("# TinyClaw Failure Log\n")
for f in failures:
lines.append(f"## {f.timestamp} — {f.task} (attempt {f.attempt})")
lines.append(f"```\n{f.error}\n```\n")
with log_path.open("a") as fh:
fh.write("\n".join(lines) + "\n")
_log(config, f" Failures logged to {log_path}")
def _execute_task(task: Task, config: RunConfig):
"""Run a single task. Raises on failure."""
if task.action == TaskAction.EDIT:
return _run_edit(task, config)
elif task.action == TaskAction.CREATE:
return _run_create(task, config)
elif task.action == TaskAction.DELETE:
return _run_delete(task)
elif task.action == TaskAction.TEST:
return _run_test(task, config)
else:
raise ValueError(f"Unknown action: {task.action}")
def _run_edit(task: Task, config: RunConfig):
"""EDIT: extract function, prompt model, verify, splice back."""
info: FunctionInfo = extract_function(task.filepath, task.target)
before = Path(task.filepath).read_text()
system, user = build_edit_prompt(info.source, task.instruction)
tokens = estimate_tokens(system + user)
if tokens > MAX_PROMPT_TOKENS:
raise ValueError(
f"Prompt too large (~{tokens} tokens). "
f"Function {task.target} may be too long for a tiny model."
)
resp = generate(
prompt=user,
model=config.model,
system=system,
api_base=config.api_base,
api_key=config.api_key,
max_tokens=config.max_tokens,
temperature=config.temperature,
)
new_code = _extract_code(resp.content)
vr = verify_syntax_string(new_code, label=f"{task.filepath}::{task.target}")
if not vr.ok:
raise ValueError(f"Model output failed syntax check: {vr.error}\n---\n{new_code[:300]}")
new_file_content = splice_function(task.filepath, info, new_code)
vr2 = verify_syntax_string(new_file_content, label=task.filepath)
if not vr2.ok:
raise ValueError(f"File failed syntax check after splice: {vr2.error}")
Path(task.filepath).write_text(new_file_content)
if config.history_path:
hist.record(
config.history_path,
action=task.action.value, filepath=task.filepath, target=task.target,
instruction=task.instruction, status="done",
before=before, after=new_file_content,
prompt_tokens=resp.prompt_tokens, completion_tokens=resp.completion_tokens,
cost=resp.cost or 0.0,
)
return resp
def _run_create(task: Task, config: RunConfig):
"""CREATE: generate new file or add new function."""
filepath = Path(task.filepath)
context = ""
if filepath.exists():
context = filepath.read_text()
system, user = build_create_prompt(task.filepath, task.instruction, context)
resp = generate(
prompt=user,
model=config.model,
system=system,
api_base=config.api_base,
api_key=config.api_key,
max_tokens=config.max_tokens,
temperature=config.temperature,
)
new_code = _extract_code(resp.content)
vr = verify_syntax_string(new_code, label=task.filepath)
if not vr.ok:
raise ValueError(f"Model output failed syntax check: {vr.error}\n---\n{new_code[:300]}")
before = context
if filepath.exists() and context.strip():
after = context.rstrip("\n") + "\n\n" + new_code + "\n"
filepath.write_text(after)
else:
filepath.parent.mkdir(parents=True, exist_ok=True)
after = new_code + "\n"
filepath.write_text(after)
if config.history_path:
hist.record(
config.history_path,
action=task.action.value, filepath=task.filepath, target=task.target,
instruction=task.instruction, status="done",
before=before, after=after,
prompt_tokens=resp.prompt_tokens, completion_tokens=resp.completion_tokens,
cost=resp.cost or 0.0,
)
return resp
def _run_delete(task: Task):
"""DELETE: remove a function from a file (no model call needed)."""
from tinyclaw.extractor import extract_function
from dataclasses import replace as dc_replace
info = extract_function(task.filepath, task.target)
lines = Path(task.filepath).read_text().splitlines(keepends=True)
new_lines = lines[: info.start_line - 1] + lines[info.end_line :]
Path(task.filepath).write_text("".join(new_lines))
# Return a fake response object
from tinyclaw.client import ModelResponse
return ModelResponse(content="", model="none", prompt_tokens=0, completion_tokens=0)
def _run_test(task: Task, config: RunConfig):
"""TEST: run pytest or a custom test command."""
import shlex, subprocess
cmd = task.instruction if task.instruction.startswith(("pytest", "python")) else f"pytest {task.filepath}"
result = subprocess.run(shlex.split(cmd), capture_output=True, text=True, timeout=60)
if result.returncode != 0:
raise ValueError(f"Test failed:\n{result.stdout[-500:]}\n{result.stderr[-500:]}")
from tinyclaw.client import ModelResponse
return ModelResponse(content="", model="none", prompt_tokens=0, completion_tokens=0)
def _extract_code(raw: str) -> str:
"""Strip markdown fences from model output."""
raw = raw.strip()
# Remove ```python ... ``` or ``` ... ```
if raw.startswith("```"):
lines = raw.splitlines()
# Drop first line (```python or ```) and last ``` if present
start = 1
end = len(lines)
if lines[-1].strip() == "```":
end -= 1
return "\n".join(lines[start:end]).strip()
return raw
def _git_commit(task: Task, config: RunConfig) -> None:
"""Auto-commit the changed file."""
msg = f"tinyclaw: {task.action.value} {task.filepath}::{task.target}\n\n{task.instruction}"
try:
subprocess.run(["git", "add", task.filepath], check=True, capture_output=True)
subprocess.run(["git", "commit", "-m", msg], check=True, capture_output=True)
except subprocess.CalledProcessError as e:
_log(config, f" [git] commit failed: {e.stderr.decode()[:200]}")
def status(queue_path: str = "QUEUE.md") -> str:
"""Return a human-readable status string for the queue."""
if not Path(queue_path).exists():
return f"No queue file at {queue_path}"
tasks = parse_queue(queue_path)
lines = [queue_summary(tasks), ""]
for t in tasks:
icon = {"done": "✓", "pending": "·", "running": "→", "failed": "✗", "skipped": "⇒"}.get(
t.status.value, "?"
)
lines.append(f" {icon} {t.action.value} {t.filepath}::{t.target} {t.instruction[:50]}")
return "\n".join(lines)
def _log(config: RunConfig, msg: str) -> None:
print(msg, flush=True)
# also push to SSE broadcast if a callback is registered
cb = getattr(config, "_progress_cb", None)
if cb:
cb({"type": "log", "msg": msg})