#!/usr/bin/env bash # Count recent web/api errors and alert without sending log text. set -euo pipefail set +x umask 077 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=notify-feishu.sh source "$script_dir/notify-feishu.sh" mode="window" case "${1:-}" in ""|--window) mode="window" ;; --daily) mode="daily" ;; *) echo "unknown argument" >&2 exit 2 ;; esac state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}" ensure_private_dir "$state_dir" env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}" load_monitor_env "$env_file" error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}" fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}" [[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2 [[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2 now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}" [[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2 if [ -n "${JYOTISHA_PYTHON:-}" ]; then py="$JYOTISHA_PYTHON" elif command -v python3 >/dev/null 2>&1; then py="python3" else py="python" fi fixture="${JYOTISHA_LOG_FIXTURE:-}" cleanup() { if [ -n "${generated_fixture:-}" ]; then rm -f -- "$generated_fixture" fi } trap cleanup EXIT generated_fixture="" if [ -z "$fixture" ]; then compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}" since="5m" [ "$mode" = "daily" ] && since="48h" generated_fixture="$(mktemp)" export APP_ENV_FILE='../.env.production' export DATABASE_ENV_FILE='../.env.production.database' export CADDYFILE_PATH='./Caddyfile.production.selfhosted' export SITE_ADDRESS='https://jyotisha.chat' sudo -n docker compose -p jyotisha-production \ --env-file "$compose_dir/.env.production" \ -f "$compose_dir/deploy/docker-compose.server.yml" \ -f "$compose_dir/deploy/docker-compose.postgres.yml" \ -f "$compose_dir/deploy/docker-compose.production.yml" \ logs --since "$since" --no-color --timestamps web api >"$generated_fixture" fixture="$generated_fixture" fi [ -f "$fixture" ] && [ ! -L "$fixture" ] || { echo "log fixture is missing" >&2 exit 1 } if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then echo "could not classify logs" >&2 exit 1 fi stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" text="" if [ -n "$rows" ]; then while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do [ -n "${service:-}" ] || continue line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}" text="${text}${line}"$'\n' done <<<"$rows" || true elif [ "$mode" = "daily" ]; then text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n' fi if [ -n "$text" ]; then now_epoch="$(date -u -d "$now_text" +%s)" [[ "$now_epoch" =~ ^[0-9]+$ ]] || exit 2 dedup_file="$state_dir/error-digest-sent.tsv" declare -A last_sent=() declare -a send_keys=() filtered="" if [ -f "$dedup_file" ] && [ ! -L "$dedup_file" ]; then while IFS=$'\t' read -r sent_epoch sent_service sent_route sent_code || [ -n "${sent_epoch:-}" ]; do [[ "${sent_epoch:-}" =~ ^[0-9]+$ ]] || continue [[ "${sent_service:-}" =~ ^[a-z0-9_-]+$ ]] || continue [ -n "${sent_route:-}" ] || continue [ -n "${sent_code:-}" ] || continue last_sent["${sent_service}"$'\t'"${sent_route}"$'\t'"${sent_code}"]="$sent_epoch" done <"$dedup_file" fi while IFS= read -r line || [ -n "$line" ]; do [ -n "$line" ] || continue if [[ "$line" =~ ^service=([a-z0-9_-]+)[[:space:]]route=([^[:space:]]+)[[:space:]]errorCode=([a-z][a-z0-9._-]*)[[:space:]] ]]; then key="${BASH_REMATCH[1]}"$'\t'"${BASH_REMATCH[2]}"$'\t'"${BASH_REMATCH[3]}" prev="${last_sent[$key]:-}" if [ -n "$prev" ] && (( now_epoch - prev < 1800 )); then continue fi send_keys+=("$key") fi filtered="${filtered}${line}"$'\n' done <<<"${text%$'\n'}" if [ -n "$filtered" ]; then post_monitor_alert "${filtered%$'\n'}" if [ "${#send_keys[@]}" -gt 0 ]; then for key in "${send_keys[@]}"; do last_sent["$key"]="$now_epoch" done fi dedup_tmp="$dedup_file.tmp.$$" : >"$dedup_tmp" for key in "${!last_sent[@]}"; do printf '%s\t%s\n' "${last_sent[$key]}" "$key" >>"$dedup_tmp" done if [ "$(uname -s)" = "Linux" ]; then chmod 600 "$dedup_tmp" fi mv "$dedup_tmp" "$dedup_file" fi fi