"""Classify fictional or container log lines into alert rows. Outbound fields are service, route, errorCode, count, and request ids. Lines with non-ASCII text, an email address, or a second calendar date are dropped so conversation text and birth dates cannot leave the host. """ from __future__ import annotations import json import re import sys from collections import OrderedDict from datetime import datetime, timedelta, timezone EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b") UUID = re.compile( r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" ) ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$") ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$") STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b") FIXTURE = re.compile( r"^(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?Pweb|api) (?P.*)$" ) DOCKER = re.compile( r"^(?Pweb|api)-\d+\s+\|\s+" r"(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P.*)$" ) CST = timezone(timedelta(hours=8)) def unsafe(line: str) -> bool: if any(ord(char) > 126 for char in line): return True lowered = line.lower() if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered: return True return len(DATE.findall(line)) >= 2 def parse_line(line: str) -> tuple[str, str, str] | None: match = FIXTURE.match(line) or DOCKER.match(line) if not match: return None return match.group("ts"), match.group("svc"), match.group("rest") def timestamp_cst_date(timestamp: str): stamp = timestamp if "." in stamp: stamp = stamp.split(".", 1)[0] + "Z" moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) return moment.astimezone(CST).date() def extract_fields(rest: str) -> tuple[str, str, str, int | None]: route = "-" error_code = "" request_id = "" status: int | None = None marker = "[agent-observability]" if marker in rest: raw = rest.split(marker, 1)[1].strip() try: payload = json.loads(raw) except json.JSONDecodeError: payload = None if isinstance(payload, dict): code = payload.get("errorCode") if isinstance(code, str) and ERROR_CODE.match(code): error_code = code candidate = payload.get("requestId") if isinstance(candidate, str) and UUID.match(candidate): request_id = candidate.lower() route_value = payload.get("route") if isinstance(route_value, str) and ROUTE.match(route_value): route = route_value status_match = STATUS.search(rest) if status_match: status = int(status_match.group(1)) return route, error_code, request_id, status def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]: now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) yesterday = now.astimezone(CST).date() - timedelta(days=1) counts: OrderedDict[tuple[str, str, str], int] = OrderedDict() request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict() fivexx_counts: OrderedDict[str, int] = OrderedDict() fivexx_ids: OrderedDict[str, list[str]] = OrderedDict() def add(key: tuple[str, str, str], request_id: str) -> None: counts[key] = counts.get(key, 0) + 1 bucket = request_ids.setdefault(key, []) if request_id and request_id not in bucket and len(bucket) < 5: bucket.append(request_id) for raw in lines: line = raw.rstrip("\r\n") if not line or unsafe(line): continue parsed = parse_line(line) if parsed is None: continue timestamp, service, rest = parsed if mode == "daily" and timestamp_cst_date(timestamp) != yesterday: continue route, error_code, request_id, status = extract_fields(rest) if error_code: add((service, route, error_code), request_id) elif status is not None: add((service, route, f"http_{status}"), request_id) if status is not None and 500 <= status <= 599: fivexx_counts[service] = fivexx_counts.get(service, 0) + 1 bucket = fivexx_ids.setdefault(service, []) if request_id and request_id not in bucket and len(bucket) < 5: bucket.append(request_id) rows: list[str] = [] def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None: shown = ",".join(ids) if ids else "-" rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}") for key, count in counts.items(): if mode == "window" and count < error_threshold: continue if count <= 0: continue emit(*key, count, request_ids.get(key, [])) for service, count in fivexx_counts.items(): if mode == "window" and count < fivexx_threshold: continue if count <= 0: continue emit(service, "-", "http_5xx", count, fivexx_ids.get(service, [])) return rows def main(argv: list[str]) -> int: if len(argv) != 5: print("usage: classify_error_log.py <5xx-threshold>", file=sys.stderr) return 2 fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv if mode not in {"window", "daily"}: print("unknown mode", file=sys.stderr) return 2 error_threshold = int(error_threshold_text) fivexx_threshold = int(fivexx_threshold_text) with open(fixture, encoding="utf-8") as handle: lines = handle.readlines() for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold): print(row) return 0 if __name__ == "__main__": raise SystemExit(main(sys.argv[1:]))