Jyotisha/deploy/monitor/classify_error_log.py

"""Classify fictional or container log lines into alert rows.

Outbound fields are service, route, errorCode, count, and request ids.
Lines with non-ASCII text, an email address, or a second calendar date are
dropped so conversation text and birth dates cannot leave the host.
"""

from __future__ import annotations

import json
import re
import sys
from collections import OrderedDict
from datetime import datetime, timedelta, timezone

EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
UUID = re.compile(
    r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
)
ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$")
ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$")
STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b")
FIXTURE = re.compile(
    r"^(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?P<svc>web|api) (?P<rest>.*)$"
)
DOCKER = re.compile(
    r"^(?P<svc>web|api)-\d+\s+\|\s+"
    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P<rest>.*)$"
)
CST = timezone(timedelta(hours=8))


def unsafe(line: str) -> bool:
    if any(ord(char) > 126 for char in line):
        return True
    lowered = line.lower()
    if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered:
        return True
    return len(DATE.findall(line)) >= 2


def parse_line(line: str) -> tuple[str, str, str] | None:
    match = FIXTURE.match(line) or DOCKER.match(line)
    if not match:
        return None
    return match.group("ts"), match.group("svc"), match.group("rest")


def timestamp_cst_date(timestamp: str):
    stamp = timestamp
    if "." in stamp:
        stamp = stamp.split(".", 1)[0] + "Z"
    moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
    return moment.astimezone(CST).date()


def extract_fields(rest: str) -> tuple[str, str, str, int | None]:
    route = "-"
    error_code = ""
    request_id = ""
    status: int | None = None
    marker = "[agent-observability]"
    if marker in rest:
        raw = rest.split(marker, 1)[1].strip()
        try:
            payload = json.loads(raw)
        except json.JSONDecodeError:
            payload = None
        if isinstance(payload, dict):
            code = payload.get("errorCode")
            if isinstance(code, str) and ERROR_CODE.match(code):
                error_code = code
            candidate = payload.get("requestId")
            if isinstance(candidate, str) and UUID.match(candidate):
                request_id = candidate.lower()
            route_value = payload.get("route")
            if isinstance(route_value, str) and ROUTE.match(route_value):
                route = route_value
    status_match = STATUS.search(rest)
    if status_match:
        status = int(status_match.group(1))
    return route, error_code, request_id, status


def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]:
    now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
    yesterday = now.astimezone(CST).date() - timedelta(days=1)
    counts: OrderedDict[tuple[str, str, str], int] = OrderedDict()
    request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict()
    fivexx_counts: OrderedDict[str, int] = OrderedDict()
    fivexx_ids: OrderedDict[str, list[str]] = OrderedDict()

    def add(key: tuple[str, str, str], request_id: str) -> None:
        counts[key] = counts.get(key, 0) + 1
        bucket = request_ids.setdefault(key, [])
        if request_id and request_id not in bucket and len(bucket) < 5:
            bucket.append(request_id)

    for raw in lines:
        line = raw.rstrip("\r\n")
        if not line or unsafe(line):
            continue
        parsed = parse_line(line)
        if parsed is None:
            continue
        timestamp, service, rest = parsed
        if mode == "daily" and timestamp_cst_date(timestamp) != yesterday:
            continue
        route, error_code, request_id, status = extract_fields(rest)
        if error_code:
            add((service, route, error_code), request_id)
        elif status is not None:
            add((service, route, f"http_{status}"), request_id)
        if status is not None and 500 <= status <= 599:
            fivexx_counts[service] = fivexx_counts.get(service, 0) + 1
            bucket = fivexx_ids.setdefault(service, [])
            if request_id and request_id not in bucket and len(bucket) < 5:
                bucket.append(request_id)

    rows: list[str] = []

    def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None:
        shown = ",".join(ids) if ids else "-"
        rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}")

    for key, count in counts.items():
        if mode == "window" and count < error_threshold:
            continue
        if count <= 0:
            continue
        emit(*key, count, request_ids.get(key, []))
    for service, count in fivexx_counts.items():
        if mode == "window" and count < fivexx_threshold:
            continue
        if count <= 0:
            continue
        emit(service, "-", "http_5xx", count, fivexx_ids.get(service, []))
    return rows


def main(argv: list[str]) -> int:
    if len(argv) != 5:
        print("usage: classify_error_log.py <fixture> <window|daily> <now> <error-threshold> <5xx-threshold>", file=sys.stderr)
        return 2
    fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv
    if mode not in {"window", "daily"}:
        print("unknown mode", file=sys.stderr)
        return 2
    error_threshold = int(error_threshold_text)
    fivexx_threshold = int(fivexx_threshold_text)
    with open(fixture, encoding="utf-8") as handle:
        lines = handle.readlines()
    for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold):
        print(row)
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv[1:]))