Jyotisha/deploy/monitor/error-digest.sh

#!/usr/bin/env bash
# Count recent web/api errors and alert without sending log text.
set -euo pipefail
set +x
umask 077

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=notify-feishu.sh
source "$script_dir/notify-feishu.sh"

mode="window"
case "${1:-}" in
  ""|--window) mode="window" ;;
  --daily) mode="daily" ;;
  *)
    echo "unknown argument" >&2
    exit 2
    ;;
esac

state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
ensure_private_dir "$state_dir"
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
load_monitor_env "$env_file"

error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2

if [ -n "${JYOTISHA_PYTHON:-}" ]; then
  py="$JYOTISHA_PYTHON"
elif command -v python3 >/dev/null 2>&1; then
  py="python3"
else
  py="python"
fi

fixture="${JYOTISHA_LOG_FIXTURE:-}"
cleanup() {
  if [ -n "${generated_fixture:-}" ]; then
    rm -f -- "$generated_fixture"
  fi
}
trap cleanup EXIT
generated_fixture=""
if [ -z "$fixture" ]; then
  compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
  since="5m"
  [ "$mode" = "daily" ] && since="48h"
  generated_fixture="$(mktemp)"
  export APP_ENV_FILE='../.env.production'
  export DATABASE_ENV_FILE='../.env.production.database'
  export CADDYFILE_PATH='./Caddyfile.production.selfhosted'
  export SITE_ADDRESS='https://jyotisha.chat'
  sudo -n docker compose -p jyotisha-production \
    --env-file "$compose_dir/.env.production" \
    -f "$compose_dir/deploy/docker-compose.server.yml" \
    -f "$compose_dir/deploy/docker-compose.postgres.yml" \
    -f "$compose_dir/deploy/docker-compose.production.yml" \
    logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
  fixture="$generated_fixture"
fi
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
  echo "log fixture is missing" >&2
  exit 1
}

if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
  echo "could not classify logs" >&2
  exit 1
fi
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
text=""
if [ -n "$rows" ]; then
  while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
    [ -n "${service:-}" ] || continue
    line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
    text="${text}${line}"$'\n'
  done <<<"$rows" || true
elif [ "$mode" = "daily" ]; then
  text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
fi

if [ -n "$text" ]; then
  now_epoch="$(date -u -d "$now_text" +%s)"
  [[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2
  dedup_file="$state_dir/error-digest-sent.tsv"
  declare -A last_sent=()
  declare -a send_keys=()
  filtered=""
  if [ -f "$dedup_file" ] && [ ! -L "$dedup_file" ]; then
    while IFS=$'\t' read -r sent_epoch sent_service sent_route sent_code || [ -n "${sent_epoch:-}" ]; do
      [[ "${sent_epoch:-}" =~ ^[0-9]+$ ]] || continue
      [[ "${sent_service:-}" =~ ^[a-z0-9_-]+$ ]] || continue
      [ -n "${sent_route:-}" ] || continue
      [ -n "${sent_code:-}" ] || continue
      last_sent["${sent_service}"$'\t'"${sent_route}"$'\t'"${sent_code}"]="$sent_epoch"
    done <"$dedup_file"
  fi
  while IFS= read -r line || [ -n "$line" ]; do
    [ -n "$line" ] || continue
    if [[ "$line" =~ ^service=([a-z0-9_-]+)[[:space:]]route=([^[:space:]]+)[[:space:]]errorCode=([a-z][a-z0-9._-]*)[[:space:]] ]]; then
      key="${BASH_REMATCH[1]}"$'\t'"${BASH_REMATCH[2]}"$'\t'"${BASH_REMATCH[3]}"
      prev="${last_sent[$key]:-}"
      if [ -n "$prev" ] && (( now_epoch - prev < 1800 )); then
        continue
      fi
      send_keys+=("$key")
    fi
    filtered="${filtered}${line}"$'\n'
  done <<<"${text%$'\n'}"
  if [ -n "$filtered" ]; then
    post_monitor_alert "${filtered%$'\n'}"
    if [ "${#send_keys[@]}" -gt 0 ]; then
      for key in "${send_keys[@]}"; do
        last_sent["$key"]="$now_epoch"
      done
    fi
    dedup_tmp="$dedup_file.tmp.$$"
    : >"$dedup_tmp"
    for key in "${!last_sent[@]}"; do
      printf '%s\t%s\n' "${last_sent[$key]}" "$key" >>"$dedup_tmp"
    done
    if [ "$(uname -s)" = "Linux" ]; then
      chmod 600 "$dedup_tmp"
    fi
    mv "$dedup_tmp" "$dedup_file"
  fi
fi