#!/usr/bin/env python3 """Matrix Homeserver Health Monitor -- tests the server from the inside out.""" import asyncio import json import logging import os import time from datetime import datetime, timezone from aiohttp import web, ClientSession, ClientTimeout logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S", ) log = logging.getLogger("matrix-monitor") # --- Config --- HOMESERVER = os.getenv("MATRIX_HOMESERVER", "https://synapse.rewr.ca") ACCESS_TOKEN = os.getenv("MATRIX_ACCESS_TOKEN", "") MONITOR_ROOM = os.getenv("MATRIX_MONITOR_ROOM", "") # !room:server or #alias:server TELEGRAM_BOT_TOKEN = os.getenv("TELEGRAM_BOT_TOKEN", "") TELEGRAM_CHAT_ID = os.getenv("TELEGRAM_CHAT_ID", "") TEST_MESSAGE_PREFIX = os.getenv("TEST_MSG_PREFIX", "[health-test]") CHECK_INTERVAL_LIGHT = int(os.getenv("CHECK_INTERVAL_LIGHT", "300")) # 5 min CHECK_INTERVAL_FULL = int(os.getenv("CHECK_INTERVAL_FULL", "3600")) # 60 min HTTP_PORT = int(os.getenv("HTTP_PORT", "8090")) REPORT_ON_FAILURE = os.getenv("REPORT_ON_FAILURE", "true").lower() == "true" REPORT_ALWAYS = os.getenv("REPORT_ALWAYS", "false").lower() == "true" API_TIMEOUT = ClientTimeout(total=30) SESSION_TIMEOUT = ClientTimeout(total=60) results: dict = { "last_run": None, "last_light": None, "last_full": None, "history": [], "consecutive_failures": 0, "current": [], } # --- Matrix API Helpers --- async def api_get(session, homeserver, path, token, timeout=None): url = f"{homeserver}/_matrix{path}" if not path.startswith("http") else path headers = {"Authorization": f"Bearer {token}"} async with session.get(url, headers=headers, timeout=(timeout or API_TIMEOUT)) as r: return r.status, await r.json() async def api_post(session, homeserver, path, token, body=None, timeout=None): url = f"{homeserver}/_matrix{path}" headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} async with session.post(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r: return r.status, await r.json() async def api_put(session, homeserver, path, token, body=None, timeout=None): url = f"{homeserver}/_matrix{path}" headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} async with session.put(url, headers=headers, json=body, timeout=(timeout or API_TIMEOUT)) as r: return r.status, await r.json() async def api_post_binary(session, homeserver, path, token, data, ctype, timeout=None): url = f"{homeserver}/_matrix{path}" headers = {"Authorization": f"Bearer {token}", "Content-Type": ctype} async with session.post(url, headers=headers, data=data, timeout=(timeout or API_TIMEOUT)) as r: return r.status, await r.json() # --- Test Suite --- async def test_auth(cs): """Verify auth token works and whoami returns valid info.""" start = time.monotonic() status, data = await api_get(cs, HOMESERVER, "/client/versions", ACCESS_TOKEN) elapsed = (time.monotonic() - start) * 1000 has_versions = status == 200 and "versions" in data start2 = time.monotonic() status2, data2 = await api_get(cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN) elapsed2 = (time.monotonic() - start2) * 1000 has_whoami = status2 == 200 and "user_id" in data2 ok = has_versions and has_whoami details = [] if has_versions: details.append(f"versions OK ({elapsed:.0f}ms)") if has_whoami: details.append(f"whoami: {data2.get('user_id')} ({elapsed2:.0f}ms)") return { "name": "Auth", "pass": ok, "latency_ms": round(elapsed + elapsed2, 1), "details": ", ".join(details) if ok else "FAILED", } async def test_sync(cs): """Measure sync response time.""" start = time.monotonic() status, data = await api_get( cs, HOMESERVER, "/client/v3/sync?timeout=1000", ACCESS_TOKEN, timeout=SESSION_TIMEOUT, ) elapsed = (time.monotonic() - start) * 1000 ok = status == 200 and "next_batch" in data return { "name": "Sync", "pass": ok, "latency_ms": round(elapsed, 1), "details": "sync OK" if ok else f"HTTP {status}", } async def test_messaging(cs): """Send a message to monitor room, measure response time.""" if not MONITOR_ROOM: return {"name": "Messaging", "pass": False, "latency_ms": 0, "details": "No monitoring room configured"} msg_id = f"msg-test-{int(time.time() * 1000)}" message = f"{TEST_MESSAGE_PREFIX} test echo {msg_id}" txn_id = f"test-{time.time()}" start = time.monotonic() status, data = await api_put( cs, HOMESERVER, f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}", ACCESS_TOKEN, {"msgtype": "m.text", "body": message}, ) send_time = (time.monotonic() - start) * 1000 if status == 200 and "event_id" in data: return { "name": "Messaging", "pass": True, "latency_ms": round(send_time, 1), "details": f"sent {data['event_id']} in {send_time:.0f}ms", } return { "name": "Messaging", "pass": False, "latency_ms": round(send_time, 1), "details": f"HTTP {status}: {data.get('error', str(data))}", } async def test_media_upload(cs): """Upload a small PNG, verify upload, try download.""" # Valid 1x1 PNG (8 bytes palette, minimal header) png_data = bytes([ 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, ]) start = time.monotonic() status, data = await api_post_binary( cs, HOMESERVER, f"/media/v3/upload?filename=monitor_test_{int(time.time())}.png", ACCESS_TOKEN, png_data, "image/png", ) elapsed = (time.monotonic() - start) * 1000 if status == 200 and "content_uri" in data: mxc = data["content_uri"] # Try downloading it back server_name_and_id = mxc.replace("mxc://", "").split("/", 1) if len(server_name_and_id) == 2: server_name, media_id = server_name_and_id url = f"{HOMESERVER}/_matrix/media/v3/download/{server_name}/{media_id}" start2 = time.monotonic() async with cs.get(url, timeout=API_TIMEOUT) as r: dl_time = (time.monotonic() - start2) * 1000 dl_ok = r.status == 200 total = elapsed + dl_time return { "name": "Media", "pass": True, "latency_ms": round(total, 1), "details": f"upload {elapsed:.0f}ms, download {dl_time:.0f}ms, download_ok={dl_ok}", } return { "name": "Media", "pass": True, "latency_ms": round(elapsed, 1), "details": f"upload OK ({elapsed:.0f}ms), download skipped (bad mxc format)", } return { "name": "Media", "pass": False, "latency_ms": round(elapsed, 1), "details": f"upload HTTP {status}: {data.get('error', str(data))}", } async def test_federation(cs): """Test federation by checking well-known and federation check.""" # Extract domain domain = HOMESERVER.split("//")[1].split("/")[0].split(":")[0] checks = [] # Well-known client start1 = time.monotonic() try: async with cs.get( f"https://{domain}/.well-known/matrix/client", timeout=API_TIMEOUT ) as r: wk_time1 = (time.monotonic() - start1) * 1000 wk_client_ok = r.status == 200 checks.append(f"well-known/client: {'OK' if wk_client_ok else f'HTTP {r.status}'} ({wk_time1:.0f}ms)") except Exception as e: wk_time1 = (time.monotonic() - start1) * 1000 wk_client_ok = False # Well-known server start2 = time.monotonic() try: async with cs.get( f"https://{domain}/.well-known/matrix/server", timeout=API_TIMEOUT ) as r: wk_time2 = (time.monotonic() - start2) * 1000 wk_server_ok = r.status == 200 checks.append(f"well-known/server: {'OK' if wk_server_ok else f'HTTP {r.status}'} ({wk_time2:.0f}ms)") except Exception as e: wk_time2 = (time.monotonic() - start2) * 1000 wk_server_ok = False # Federation check via matrix.org's API start3 = time.monotonic() try: async with cs.get( f"https://matrix.org/federationtester/api/report?server_name={domain}", timeout=ClientTimeout(total=10), ) as r: fedi_time = (time.monotonic() - start3) * 1000 if r.status == 200: fed_data = await r.json() all_ok = fed_data.get("AllChecksOK", False) checks.append(f"federation: {'PASS' if all_ok else 'WARN'} ({fedi_time:.0f}ms)") else: checks.append(f"federation: HTTP {r.status} ({fedi_time:.0f}ms)") except Exception as e: fedi_time = (time.monotonic() - start3) * 1000 checks.append(f"federation: ERROR ({fedi_time:.0f}ms): {str(e)[:80]}") total = sum(t for t in [wk_time1, wk_time2] if t > 0) + fedi_time all_pass = wk_client_ok and wk_server_ok return { "name": "Federation", "pass": all_pass, "latency_ms": round(total, 1), "details": "; ".join(checks), } async def test_rooms(cs): """Create a room, verify state, leave it.""" start = time.monotonic() status, data = await api_post( cs, HOMESERVER, "/client/v3/createRoom", ACCESS_TOKEN, { "preset": "private_chat", "name": f"monitor-test-{int(time.time())}", }, ) create_time = (time.monotonic() - start) * 1000 if status != 200 or "room_id" not in data: return { "name": "Rooms", "pass": False, "latency_ms": round(create_time, 1), "details": f"create failed: HTTP {status}", } room_id = data["room_id"] ok = True details = [f"created {room_id} ({create_time:.0f}ms)"] # Check room state start2 = time.monotonic() status2, data2 = await api_get( cs, HOMESERVER, f"/client/v3/rooms/{room_id}/state", ACCESS_TOKEN ) state_time = (time.monotonic() - start2) * 1000 if status2 == 200: details.append(f"state OK ({state_time:.0f}ms, {len(data2)} events)") else: ok = False details.append(f"state HTTP {status2}") # Leave the room status3, _ = await api_post( cs, HOMESERVER, f"/client/v3/rooms/{room_id}/leave", ACCESS_TOKEN, {} ) if status3 != 200: ok = False details.append(f"leave HTTP {status3}") else: details.append("left OK") total = create_time + state_time return { "name": "Rooms", "pass": ok, "latency_ms": round(total, 1), "details": "; ".join(details), } async def test_presence(cs): """Set and verify presence state.""" start = time.monotonic() status, _ = await api_put( cs, HOMESERVER, "/client/v3/presence/@monitor:rewr.ca/status", ACCESS_TOKEN, {"presence": "online", "status_msg": "monitoring"}, ) elapsed = (time.monotonic() - start) * 1000 # Try getting own user ID for presence check status2, data2 = await api_get( cs, HOMESERVER, "/client/v3/account/whoami", ACCESS_TOKEN ) if status2 == 200 and "user_id" in data2: user_id = data2["user_id"] import urllib.parse encoded_id = urllib.parse.quote(user_id) status3, _ = await api_get( cs, HOMESERVER, f"/client/v3/presence/{encoded_id}/status", ACCESS_TOKEN ) if status3 == 200: ok = status == 200 return { "name": "Presence", "pass": ok, "latency_ms": round(elapsed, 1), "details": f"presence set + fetch OK ({elapsed:.0f}ms)", } return { "name": "Presence", "pass": False, "latency_ms": round(elapsed, 1), "details": f"set OK ({elapsed:.0f}ms), fetch HTTP {status3}", } return { "name": "Presence", "pass": status == 200, "latency_ms": round(elapsed, 1), "details": f"presence set" if status == 200 else f"HTTP {status}", } # --- Room Management --- async def ensure_monitor_room(cs): """Create the monitor room if not configured.""" global MONITOR_ROOM if MONITOR_ROOM: log.info(f"Using configured monitor room: {MONITOR_ROOM}") return log.info("No monitor room configured, creating one...") status, data = await api_post( cs, HOMESERVER, "/client/v3/createRoom", ACCESS_TOKEN, { "name": "Matrix Health Monitor", "topic": "Automated health check results from Matrix-Health-Monitor", "initial_state": [ { "type": "m.room.history_visibility", "state_key": "", "content": {"history_visibility": "shared"}, }, { "type": "m.room.guest_access", "state_key": "", "content": {"guest_access": "forbidden"}, }, ], }, ) if status == 200 and "room_id" in data: MONITOR_ROOM = data["room_id"] log.info(f"Created monitor room: {MONITOR_ROOM}") else: log.error(f"Failed to create monitor room: {status} {data}") # --- Report --- def build_report(test_results, run_type="full"): lines = [] ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") lines.append(f"Matrix Health Check ({run_type}) -- {ts}") lines.append("") all_pass = True for r in test_results: icon = "PASS" if r["pass"] else "FAIL" if not r["pass"]: all_pass = False lines.append(f" [{icon}] {r['name']} -- {r['latency_ms']}ms -- {r['details']}") summary = "ALL GREEN" if all_pass else "ISSUES DETECTED" lines.insert(1, f"Summary: {summary}") if not all_pass: failed = [r["name"] for r in test_results if not r["pass"]] lines.insert(2, f"Failed: {', '.join(failed)}") return "\n".join(lines), all_pass async def send_matrix_report(text): """Send a report to the monitor room.""" if not MONITOR_ROOM: return async with ClientSession(timeout=API_TIMEOUT) as cs: txn_id = f"report-{time.time()}" await api_put( cs, HOMESERVER, f"/client/v3/rooms/{MONITOR_ROOM}/send/m.room.message/{txn_id}", ACCESS_TOKEN, { "msgtype": "m.text", "body": text, "format": "org.matrix.custom.html", "formatted_body": text.replace("\n", "
").replace(" ", "  "), }, ) async def send_telegram_alert(text): """Send an alert to Telegram.""" if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID: return url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage" try: async with ClientSession(timeout=ClientTimeout(total=10)) as cs: await cs.post(url, json={ "chat_id": TELEGRAM_CHAT_ID, "text": text, "parse_mode": "Markdown", }) except Exception as e: log.error(f"Telegram alert failed: {e}") # --- Test Runners --- async def run_light_test(): """Lightweight: auth, sync, messaging.""" async with ClientSession(timeout=API_TIMEOUT) as cs: auth = await test_auth(cs) sync = await test_sync(cs) msg = await test_messaging(cs) return [auth, sync, msg] async def run_full_test(): """Full suite: everything.""" async with ClientSession(timeout=API_TIMEOUT) as cs: auth = await test_auth(cs) sync = await test_sync(cs) msg = await test_messaging(cs) media = await test_media_upload(cs) fedi = await test_federation(cs) rooms = await test_rooms(cs) presence = await test_presence(cs) return [auth, sync, msg, media, fedi, rooms, presence] async def run_and_report(run_type="full"): if run_type == "full": tests = await run_full_test() else: tests = await run_light_test() results["last_run"] = datetime.now(timezone.utc).isoformat() results["last_light" if run_type == "light" else "last_full"] = results["last_run"] results["history"].append({ "time": results["last_run"], "type": run_type, "results": tests, }) if len(results["history"]) > 100: results["history"] = results["history"][-100:] report_text, all_pass = build_report(tests, run_type) # Save log log_file = f"/data/health-{datetime.now(timezone.utc).strftime('%Y-%m-%d')}.json" try: with open(log_file, "w") as f: json.dump({ "time": results["last_run"], "type": run_type, "tests": tests, }, f, indent=2) except Exception as e: log.error(f"Failed to write log: {e}") # Track consecutive failures if all_pass: results["consecutive_failures"] = 0 else: results["consecutive_failures"] += 1 results["current"] = tests # Report logic failed_tests = [t for t in tests if not t["pass"]] if all_pass and REPORT_ALWAYS: await send_matrix_report(report_text) elif not all_pass: log.warning(f"Health check ({run_type}) failed: {[t['name'] for t in failed_tests]}") await send_matrix_report(report_text) if results["consecutive_failures"] >= 2: await send_telegram_alert( f"Matrix Health Alert ({run_type}): " f"{len(failed_tests)} tests failing -- " f"{', '.join(t['name'] for t in failed_tests)}" ) else: log.info(f"Health check ({run_type}) passed") return tests # --- HTTP Server --- async def handle_health(request): current = results.get("current", []) if not current: return web.json_response({"status": "no data yet"}, 200) all_pass = all(t["pass"] for t in current) status_code = 200 if all_pass else 503 return web.json_response({ "status": "healthy" if all_pass else "degraded", "last_run": results.get("last_run"), "tests": {t["name"]: { "pass": t["pass"], "latency_ms": t["latency_ms"], } for t in current}, "consecutive_failures": results.get("consecutive_failures", 0), }, status=status_code) async def handle_status(request): return web.json_response({ "last_run": results.get("last_run"), "last_light": results.get("last_light"), "last_full": results.get("last_full"), "current": results.get("current", []), "history_count": len(results.get("history", [])), "consecutive_failures": results.get("consecutive_failures", 0), }) # --- Main --- async def monitoring_loop(): log.info(f"Health check intervals: light={CHECK_INTERVAL_LIGHT}s, full={CHECK_INTERVAL_FULL}s") # Run initial full check log.info("Running initial full health check...") await run_and_report("full") last_light = time.monotonic() last_full = time.monotonic() while True: await asyncio.sleep(10) now = time.monotonic() if now - last_full >= CHECK_INTERVAL_FULL: log.info("Running full health check...") await run_and_report("full") last_full = now last_light = now elif now - last_light >= CHECK_INTERVAL_LIGHT: log.info("Running light health check...") await run_and_report("light") last_light = now async def start_http_server(): app = web.Application() app.router.add_get("/health", handle_health) app.router.add_get("/status", handle_status) runner = web.AppRunner(app) await runner.setup() site = web.TCPSite(runner, "0.0.0.0", HTTP_PORT) await site.start() log.info(f"HTTP health endpoint on port {HTTP_PORT}") async def main(): if not ACCESS_TOKEN: log.error("MATRIX_ACCESS_TOKEN is required (env: MATRIX_ACCESS_TOKEN)") return # Ensure monitor room exists async with ClientSession(timeout=API_TIMEOUT) as cs: await ensure_monitor_room(cs) log.info(f"Matrix Health Monitor starting") log.info(f"Homeserver: {HOMESERVER}") log.info(f"Monitor room: {MONITOR_ROOM or 'none (Matrix reports disabled)'}") log.info(f"Telegram alerts: {'enabled' if TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID else 'disabled'}") await asyncio.gather( start_http_server(), monitoring_loop(), ) if __name__ == "__main__": asyncio.run(main())