Web UI (tinyclaw web): - Flask + SSE real-time progress streaming - Queue view: add/delete tasks, status colours, error display - Live log panel with coloured output - History tab with unified diffs (Rosé Pine theme) - Model picker dropdown - Auto-refresh every 5s History (history.py): - JSONL-based, records before/after content + unified diff - Wired into driver for EDIT and CREATE actions - Failed tasks also recorded QA benchmark (tinyclaw qa <models...>): - 5 fixed tasks: divide/greet/is_even/repeat/safe_get - Scores syntax validity + instruction following - Comparison table: pass%, syntax%, latency, cost QA results (3 models): - gemma-3-12b-it: 80% pass, 100% syntax, ~944ms ← winner - gemma-3-4b-it: 60% pass, 100% syntax, ~917ms - qwen2.5-coder-7b: 0% pass, 0% syntax (max_tokens issue — hits 512 limit mid-function)
364 lines
12 KiB
Python
364 lines
12 KiB
Python
"""Core driver loop: read queue → extract → prompt → call model → verify → commit."""
|
|
|
|
import os
|
|
import subprocess
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
def _load_dotenv(path: str = ".env") -> None:
|
|
"""Load KEY=VALUE pairs from .env into os.environ (no-op if missing)."""
|
|
p = Path(path)
|
|
if not p.exists():
|
|
return
|
|
for line in p.read_text().splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, val = line.partition("=")
|
|
os.environ.setdefault(key.strip(), val.strip())
|
|
|
|
from tinyclaw import history as hist
|
|
from tinyclaw.client import generate
|
|
from tinyclaw.extractor import extract_function, splice_function, FunctionInfo
|
|
from tinyclaw.prompt import build_edit_prompt, build_create_prompt, estimate_tokens
|
|
from tinyclaw.queue import (
|
|
Task, TaskAction, TaskStatus,
|
|
parse_queue, write_queue, queue_summary,
|
|
)
|
|
from tinyclaw.verifier import verify_syntax_string, verify_function_present
|
|
|
|
|
|
MAX_RETRIES = 3
|
|
MAX_PROMPT_TOKENS = 600 # warn if exceeded
|
|
|
|
|
|
@dataclass
|
|
class RunConfig:
|
|
queue_path: str = "QUEUE.md"
|
|
model: str = "qwen/qwen3.5-1.5b"
|
|
api_base: str | None = None
|
|
api_key: str | None = None
|
|
dry_run: bool = False
|
|
verbose: bool = False
|
|
max_tokens: int = 1024
|
|
temperature: float = 0.1
|
|
auto_commit: bool = True
|
|
history_path: str = "HISTORY.jsonl"
|
|
|
|
|
|
@dataclass
|
|
class FailureEntry:
|
|
task: str
|
|
attempt: int
|
|
error: str
|
|
timestamp: str
|
|
|
|
|
|
@dataclass
|
|
class RunResult:
|
|
done: int = 0
|
|
failed: int = 0
|
|
skipped: int = 0
|
|
total_cost: float = 0.0
|
|
total_prompt_tokens: int = 0
|
|
total_completion_tokens: int = 0
|
|
failures: list[FailureEntry] = field(default_factory=list)
|
|
|
|
|
|
def run(config: RunConfig) -> RunResult:
|
|
"""Process the task queue. Returns a summary of results."""
|
|
_load_dotenv()
|
|
|
|
queue_path = config.queue_path
|
|
|
|
if not Path(queue_path).exists():
|
|
raise FileNotFoundError(f"Queue file not found: {queue_path}")
|
|
|
|
tasks = parse_queue(queue_path)
|
|
pending = [t for t in tasks if t.status == TaskStatus.PENDING]
|
|
|
|
_log(config, f"Queue: {queue_summary(tasks)}")
|
|
_log(config, f"Model: {config.model}")
|
|
|
|
if config.dry_run:
|
|
_log(config, "[dry-run] Tasks that would run:")
|
|
for t in pending:
|
|
_log(config, f" {t.action.value} {t.filepath}::{t.target} — {t.instruction}")
|
|
return RunResult()
|
|
|
|
result = RunResult()
|
|
|
|
for task in pending:
|
|
_log(config, f"\n→ {task.action.value} {task.filepath}::{task.target}")
|
|
_log(config, f" {task.instruction}")
|
|
|
|
success = False
|
|
for attempt in range(1, MAX_RETRIES + 1):
|
|
task.attempts = attempt
|
|
task.status = TaskStatus.RUNNING
|
|
|
|
try:
|
|
resp = _execute_task(task, config)
|
|
result.total_cost += resp.cost or 0.0
|
|
result.total_prompt_tokens += resp.prompt_tokens
|
|
result.total_completion_tokens += resp.completion_tokens
|
|
task.status = TaskStatus.DONE
|
|
success = True
|
|
_log(config, f" ✓ attempt {attempt} — {resp.prompt_tokens}pt / {resp.completion_tokens}ct")
|
|
break
|
|
|
|
except Exception as e:
|
|
task.error = str(e)
|
|
_log(config, f" ✗ attempt {attempt}: {e}")
|
|
result.failures.append(FailureEntry(
|
|
task=f"{task.action.value} {task.filepath}::{task.target}",
|
|
attempt=attempt,
|
|
error=str(e),
|
|
timestamp=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
|
|
))
|
|
if attempt < MAX_RETRIES:
|
|
import time
|
|
# Exponential backoff: 2s, 4s — longer on rate limit
|
|
wait = (2 ** attempt) * (5 if "429" in str(e) else 1)
|
|
_log(config, f" retrying in {wait}s...")
|
|
time.sleep(wait)
|
|
|
|
if not success:
|
|
if task.attempts >= MAX_RETRIES:
|
|
task.status = TaskStatus.SKIPPED
|
|
_log(config, f" → skipped after {MAX_RETRIES} attempts")
|
|
result.skipped += 1
|
|
else:
|
|
task.status = TaskStatus.FAILED
|
|
result.failed += 1
|
|
if config.history_path:
|
|
hist.record(
|
|
config.history_path,
|
|
action=task.action.value, filepath=task.filepath, target=task.target,
|
|
instruction=task.instruction, status=task.status.value,
|
|
error=task.error,
|
|
)
|
|
else:
|
|
result.done += 1
|
|
|
|
# Persist queue state after every task
|
|
write_queue(queue_path, tasks)
|
|
|
|
# Auto-commit on success
|
|
if success and config.auto_commit:
|
|
_git_commit(task, config)
|
|
|
|
_log(config, f"\nDone: {result.done} ✓ Failed: {result.failed} ✗ Skipped: {result.skipped} →")
|
|
if result.total_cost:
|
|
_log(config, f"Cost: ${result.total_cost:.4f} Tokens: {result.total_prompt_tokens}p / {result.total_completion_tokens}c")
|
|
|
|
if result.failures:
|
|
_write_failure_log(result.failures, config)
|
|
|
|
return result
|
|
|
|
|
|
def _write_failure_log(failures: list[FailureEntry], config: RunConfig) -> None:
|
|
"""Append failures to FAILURES.md next to the queue file."""
|
|
log_path = Path(config.queue_path).parent / "FAILURES.md"
|
|
lines = []
|
|
if not log_path.exists():
|
|
lines.append("# TinyClaw Failure Log\n")
|
|
for f in failures:
|
|
lines.append(f"## {f.timestamp} — {f.task} (attempt {f.attempt})")
|
|
lines.append(f"```\n{f.error}\n```\n")
|
|
with log_path.open("a") as fh:
|
|
fh.write("\n".join(lines) + "\n")
|
|
_log(config, f" Failures logged to {log_path}")
|
|
|
|
|
|
def _execute_task(task: Task, config: RunConfig):
|
|
"""Run a single task. Raises on failure."""
|
|
if task.action == TaskAction.EDIT:
|
|
return _run_edit(task, config)
|
|
elif task.action == TaskAction.CREATE:
|
|
return _run_create(task, config)
|
|
elif task.action == TaskAction.DELETE:
|
|
return _run_delete(task)
|
|
elif task.action == TaskAction.TEST:
|
|
return _run_test(task, config)
|
|
else:
|
|
raise ValueError(f"Unknown action: {task.action}")
|
|
|
|
|
|
def _run_edit(task: Task, config: RunConfig):
|
|
"""EDIT: extract function, prompt model, verify, splice back."""
|
|
info: FunctionInfo = extract_function(task.filepath, task.target)
|
|
before = Path(task.filepath).read_text()
|
|
|
|
system, user = build_edit_prompt(info.source, task.instruction)
|
|
|
|
tokens = estimate_tokens(system + user)
|
|
if tokens > MAX_PROMPT_TOKENS:
|
|
raise ValueError(
|
|
f"Prompt too large (~{tokens} tokens). "
|
|
f"Function {task.target} may be too long for a tiny model."
|
|
)
|
|
|
|
resp = generate(
|
|
prompt=user,
|
|
model=config.model,
|
|
system=system,
|
|
api_base=config.api_base,
|
|
api_key=config.api_key,
|
|
max_tokens=config.max_tokens,
|
|
temperature=config.temperature,
|
|
)
|
|
|
|
new_code = _extract_code(resp.content)
|
|
|
|
vr = verify_syntax_string(new_code, label=f"{task.filepath}::{task.target}")
|
|
if not vr.ok:
|
|
raise ValueError(f"Model output failed syntax check: {vr.error}\n---\n{new_code[:300]}")
|
|
|
|
new_file_content = splice_function(task.filepath, info, new_code)
|
|
|
|
vr2 = verify_syntax_string(new_file_content, label=task.filepath)
|
|
if not vr2.ok:
|
|
raise ValueError(f"File failed syntax check after splice: {vr2.error}")
|
|
|
|
Path(task.filepath).write_text(new_file_content)
|
|
|
|
if config.history_path:
|
|
hist.record(
|
|
config.history_path,
|
|
action=task.action.value, filepath=task.filepath, target=task.target,
|
|
instruction=task.instruction, status="done",
|
|
before=before, after=new_file_content,
|
|
prompt_tokens=resp.prompt_tokens, completion_tokens=resp.completion_tokens,
|
|
cost=resp.cost or 0.0,
|
|
)
|
|
|
|
return resp
|
|
|
|
|
|
def _run_create(task: Task, config: RunConfig):
|
|
"""CREATE: generate new file or add new function."""
|
|
filepath = Path(task.filepath)
|
|
|
|
context = ""
|
|
if filepath.exists():
|
|
context = filepath.read_text()
|
|
|
|
system, user = build_create_prompt(task.filepath, task.instruction, context)
|
|
|
|
resp = generate(
|
|
prompt=user,
|
|
model=config.model,
|
|
system=system,
|
|
api_base=config.api_base,
|
|
api_key=config.api_key,
|
|
max_tokens=config.max_tokens,
|
|
temperature=config.temperature,
|
|
)
|
|
|
|
new_code = _extract_code(resp.content)
|
|
|
|
vr = verify_syntax_string(new_code, label=task.filepath)
|
|
if not vr.ok:
|
|
raise ValueError(f"Model output failed syntax check: {vr.error}\n---\n{new_code[:300]}")
|
|
|
|
before = context
|
|
if filepath.exists() and context.strip():
|
|
after = context.rstrip("\n") + "\n\n" + new_code + "\n"
|
|
filepath.write_text(after)
|
|
else:
|
|
filepath.parent.mkdir(parents=True, exist_ok=True)
|
|
after = new_code + "\n"
|
|
filepath.write_text(after)
|
|
|
|
if config.history_path:
|
|
hist.record(
|
|
config.history_path,
|
|
action=task.action.value, filepath=task.filepath, target=task.target,
|
|
instruction=task.instruction, status="done",
|
|
before=before, after=after,
|
|
prompt_tokens=resp.prompt_tokens, completion_tokens=resp.completion_tokens,
|
|
cost=resp.cost or 0.0,
|
|
)
|
|
|
|
return resp
|
|
|
|
|
|
def _run_delete(task: Task):
|
|
"""DELETE: remove a function from a file (no model call needed)."""
|
|
from tinyclaw.extractor import extract_function
|
|
from dataclasses import replace as dc_replace
|
|
|
|
info = extract_function(task.filepath, task.target)
|
|
lines = Path(task.filepath).read_text().splitlines(keepends=True)
|
|
new_lines = lines[: info.start_line - 1] + lines[info.end_line :]
|
|
Path(task.filepath).write_text("".join(new_lines))
|
|
|
|
# Return a fake response object
|
|
from tinyclaw.client import ModelResponse
|
|
return ModelResponse(content="", model="none", prompt_tokens=0, completion_tokens=0)
|
|
|
|
|
|
def _run_test(task: Task, config: RunConfig):
|
|
"""TEST: run pytest or a custom test command."""
|
|
import shlex, subprocess
|
|
cmd = task.instruction if task.instruction.startswith(("pytest", "python")) else f"pytest {task.filepath}"
|
|
result = subprocess.run(shlex.split(cmd), capture_output=True, text=True, timeout=60)
|
|
if result.returncode != 0:
|
|
raise ValueError(f"Test failed:\n{result.stdout[-500:]}\n{result.stderr[-500:]}")
|
|
|
|
from tinyclaw.client import ModelResponse
|
|
return ModelResponse(content="", model="none", prompt_tokens=0, completion_tokens=0)
|
|
|
|
|
|
def _extract_code(raw: str) -> str:
|
|
"""Strip markdown fences from model output."""
|
|
raw = raw.strip()
|
|
|
|
# Remove ```python ... ``` or ``` ... ```
|
|
if raw.startswith("```"):
|
|
lines = raw.splitlines()
|
|
# Drop first line (```python or ```) and last ``` if present
|
|
start = 1
|
|
end = len(lines)
|
|
if lines[-1].strip() == "```":
|
|
end -= 1
|
|
return "\n".join(lines[start:end]).strip()
|
|
|
|
return raw
|
|
|
|
|
|
def _git_commit(task: Task, config: RunConfig) -> None:
|
|
"""Auto-commit the changed file."""
|
|
msg = f"tinyclaw: {task.action.value} {task.filepath}::{task.target}\n\n{task.instruction}"
|
|
try:
|
|
subprocess.run(["git", "add", task.filepath], check=True, capture_output=True)
|
|
subprocess.run(["git", "commit", "-m", msg], check=True, capture_output=True)
|
|
except subprocess.CalledProcessError as e:
|
|
_log(config, f" [git] commit failed: {e.stderr.decode()[:200]}")
|
|
|
|
|
|
def status(queue_path: str = "QUEUE.md") -> str:
|
|
"""Return a human-readable status string for the queue."""
|
|
if not Path(queue_path).exists():
|
|
return f"No queue file at {queue_path}"
|
|
tasks = parse_queue(queue_path)
|
|
lines = [queue_summary(tasks), ""]
|
|
for t in tasks:
|
|
icon = {"done": "✓", "pending": "·", "running": "→", "failed": "✗", "skipped": "⇒"}.get(
|
|
t.status.value, "?"
|
|
)
|
|
lines.append(f" {icon} {t.action.value} {t.filepath}::{t.target} {t.instruction[:50]}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _log(config: RunConfig, msg: str) -> None:
|
|
print(msg, flush=True)
|
|
# also push to SSE broadcast if a callback is registered
|
|
cb = getattr(config, "_progress_cb", None)
|
|
if cb:
|
|
cb({"type": "log", "msg": msg})
|