86 lines
2.7 KiB
Bash
86 lines
2.7 KiB
Bash
#!/usr/bin/env bash
|
|
# Count recent web/api errors and alert without sending log text.
|
|
set -euo pipefail
|
|
set +x
|
|
umask 077
|
|
|
|
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
# shellcheck source=notify-feishu.sh
|
|
source "$script_dir/notify-feishu.sh"
|
|
|
|
mode="window"
|
|
case "${1:-}" in
|
|
""|--window) mode="window" ;;
|
|
--daily) mode="daily" ;;
|
|
*)
|
|
echo "unknown argument" >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
|
ensure_private_dir "$state_dir"
|
|
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
|
load_monitor_env "$env_file"
|
|
|
|
error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
|
|
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
|
|
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
|
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
|
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
|
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2
|
|
|
|
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
|
py="$JYOTISHA_PYTHON"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
py="python3"
|
|
else
|
|
py="python"
|
|
fi
|
|
|
|
fixture="${JYOTISHA_LOG_FIXTURE:-}"
|
|
cleanup() {
|
|
if [ -n "${generated_fixture:-}" ]; then
|
|
rm -f -- "$generated_fixture"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
generated_fixture=""
|
|
if [ -z "$fixture" ]; then
|
|
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
|
since="5m"
|
|
[ "$mode" = "daily" ] && since="48h"
|
|
generated_fixture="$(mktemp)"
|
|
docker compose -p jyotisha-production \
|
|
--env-file "$compose_dir/.env.production" \
|
|
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
|
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
|
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
|
logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
|
|
fixture="$generated_fixture"
|
|
fi
|
|
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
|
|
echo "log fixture is missing" >&2
|
|
exit 1
|
|
}
|
|
|
|
if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
|
|
echo "could not classify logs" >&2
|
|
exit 1
|
|
fi
|
|
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
text=""
|
|
if [ -n "$rows" ]; then
|
|
while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
|
|
[ -n "${service:-}" ] || continue
|
|
line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
|
|
text="${text}${line}"$'\n'
|
|
done <<<"$rows" || true
|
|
elif [ "$mode" = "daily" ]; then
|
|
text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
|
|
fi
|
|
|
|
if [ -n "$text" ]; then
|
|
post_monitor_alert "${text%$'\n'}"
|
|
fi
|