给内测补上每日加密备份、宕机告警和日志上限。
主机安装和一次恢复演练留给产品。页面没有变化。 Co-Authored-By: Grok (agent loop)
This commit is contained in:
co-authored by
Grok (agent loop)
parent
71f7fac3a1
commit
53d8df257d
@@ -0,0 +1,160 @@
|
||||
"""Classify fictional or container log lines into alert rows.
|
||||
|
||||
Outbound fields are service, route, errorCode, count, and request ids.
|
||||
Lines with non-ASCII text, an email address, or a second calendar date are
|
||||
dropped so conversation text and birth dates cannot leave the host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
DATE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b")
|
||||
UUID = re.compile(
|
||||
r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$"
|
||||
)
|
||||
ERROR_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,79}$")
|
||||
ROUTE = re.compile(r"^/[A-Za-z0-9/_.-]{0,120}$")
|
||||
STATUS = re.compile(r"(?:statusCode|status)[\"']?\s*[:=]\s*(\d{3})\b")
|
||||
FIXTURE = re.compile(
|
||||
r"^(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?P<svc>web|api) (?P<rest>.*)$"
|
||||
)
|
||||
DOCKER = re.compile(
|
||||
r"^(?P<svc>web|api)-\d+\s+\|\s+"
|
||||
r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) (?P<rest>.*)$"
|
||||
)
|
||||
CST = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def unsafe(line: str) -> bool:
|
||||
if any(ord(char) > 126 for char in line):
|
||||
return True
|
||||
lowered = line.lower()
|
||||
if EMAIL.search(line) or "birthdate" in lowered or "birth_date" in lowered:
|
||||
return True
|
||||
return len(DATE.findall(line)) >= 2
|
||||
|
||||
|
||||
def parse_line(line: str) -> tuple[str, str, str] | None:
|
||||
match = FIXTURE.match(line) or DOCKER.match(line)
|
||||
if not match:
|
||||
return None
|
||||
return match.group("ts"), match.group("svc"), match.group("rest")
|
||||
|
||||
|
||||
def timestamp_cst_date(timestamp: str):
|
||||
stamp = timestamp
|
||||
if "." in stamp:
|
||||
stamp = stamp.split(".", 1)[0] + "Z"
|
||||
moment = datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
return moment.astimezone(CST).date()
|
||||
|
||||
|
||||
def extract_fields(rest: str) -> tuple[str, str, str, int | None]:
|
||||
route = "-"
|
||||
error_code = ""
|
||||
request_id = ""
|
||||
status: int | None = None
|
||||
marker = "[agent-observability]"
|
||||
if marker in rest:
|
||||
raw = rest.split(marker, 1)[1].strip()
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
payload = None
|
||||
if isinstance(payload, dict):
|
||||
code = payload.get("errorCode")
|
||||
if isinstance(code, str) and ERROR_CODE.match(code):
|
||||
error_code = code
|
||||
candidate = payload.get("requestId")
|
||||
if isinstance(candidate, str) and UUID.match(candidate):
|
||||
request_id = candidate.lower()
|
||||
route_value = payload.get("route")
|
||||
if isinstance(route_value, str) and ROUTE.match(route_value):
|
||||
route = route_value
|
||||
status_match = STATUS.search(rest)
|
||||
if status_match:
|
||||
status = int(status_match.group(1))
|
||||
return route, error_code, request_id, status
|
||||
|
||||
|
||||
def classify(lines: list[str], mode: str, now_text: str, error_threshold: int, fivexx_threshold: int) -> list[str]:
|
||||
now = datetime.strptime(now_text, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
yesterday = now.astimezone(CST).date() - timedelta(days=1)
|
||||
counts: OrderedDict[tuple[str, str, str], int] = OrderedDict()
|
||||
request_ids: OrderedDict[tuple[str, str, str], list[str]] = OrderedDict()
|
||||
fivexx_counts: OrderedDict[str, int] = OrderedDict()
|
||||
fivexx_ids: OrderedDict[str, list[str]] = OrderedDict()
|
||||
|
||||
def add(key: tuple[str, str, str], request_id: str) -> None:
|
||||
counts[key] = counts.get(key, 0) + 1
|
||||
bucket = request_ids.setdefault(key, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
for raw in lines:
|
||||
line = raw.rstrip("\r\n")
|
||||
if not line or unsafe(line):
|
||||
continue
|
||||
parsed = parse_line(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
timestamp, service, rest = parsed
|
||||
if mode == "daily" and timestamp_cst_date(timestamp) != yesterday:
|
||||
continue
|
||||
route, error_code, request_id, status = extract_fields(rest)
|
||||
if error_code:
|
||||
add((service, route, error_code), request_id)
|
||||
elif status is not None:
|
||||
add((service, route, f"http_{status}"), request_id)
|
||||
if status is not None and 500 <= status <= 599:
|
||||
fivexx_counts[service] = fivexx_counts.get(service, 0) + 1
|
||||
bucket = fivexx_ids.setdefault(service, [])
|
||||
if request_id and request_id not in bucket and len(bucket) < 5:
|
||||
bucket.append(request_id)
|
||||
|
||||
rows: list[str] = []
|
||||
|
||||
def emit(service: str, route: str, code: str, count: int, ids: list[str]) -> None:
|
||||
shown = ",".join(ids) if ids else "-"
|
||||
rows.append(f"{service}\t{route}\t{code}\t{count}\t{shown}")
|
||||
|
||||
for key, count in counts.items():
|
||||
if mode == "window" and count < error_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(*key, count, request_ids.get(key, []))
|
||||
for service, count in fivexx_counts.items():
|
||||
if mode == "window" and count < fivexx_threshold:
|
||||
continue
|
||||
if count <= 0:
|
||||
continue
|
||||
emit(service, "-", "http_5xx", count, fivexx_ids.get(service, []))
|
||||
return rows
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 5:
|
||||
print("usage: classify_error_log.py <fixture> <window|daily> <now> <error-threshold> <5xx-threshold>", file=sys.stderr)
|
||||
return 2
|
||||
fixture, mode, now_text, error_threshold_text, fivexx_threshold_text = argv
|
||||
if mode not in {"window", "daily"}:
|
||||
print("unknown mode", file=sys.stderr)
|
||||
return 2
|
||||
error_threshold = int(error_threshold_text)
|
||||
fivexx_threshold = int(fivexx_threshold_text)
|
||||
with open(fixture, encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
for row in classify(lines, mode, now_text, error_threshold, fivexx_threshold):
|
||||
print(row)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Count recent web/api errors and alert without sending log text.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="window"
|
||||
case "${1:-}" in
|
||||
""|--window) mode="window" ;;
|
||||
--daily) mode="daily" ;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
error_threshold="${JYOTISHA_ERROR_THRESHOLD:-10}"
|
||||
fivexx_threshold="${JYOTISHA_5XX_THRESHOLD:-20}"
|
||||
[[ "$error_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
[[ "$fivexx_threshold" =~ ^[1-9][0-9]{0,4}$ ]] || exit 2
|
||||
now_text="${JYOTISHA_NOW:-$(date -u +%Y-%m-%dT%H:%M:%SZ)}"
|
||||
[[ "$now_text" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]] || exit 2
|
||||
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
|
||||
fixture="${JYOTISHA_LOG_FIXTURE:-}"
|
||||
cleanup() {
|
||||
if [ -n "${generated_fixture:-}" ]; then
|
||||
rm -f -- "$generated_fixture"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
generated_fixture=""
|
||||
if [ -z "$fixture" ]; then
|
||||
compose_dir="${JYOTISHA_COMPOSE_DIR:-/opt/jyotisha-production}"
|
||||
since="5m"
|
||||
[ "$mode" = "daily" ] && since="48h"
|
||||
generated_fixture="$(mktemp)"
|
||||
docker compose -p jyotisha-production \
|
||||
--env-file "$compose_dir/.env.production" \
|
||||
-f "$compose_dir/deploy/docker-compose.server.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.postgres.yml" \
|
||||
-f "$compose_dir/deploy/docker-compose.production.yml" \
|
||||
logs --since "$since" --no-color --timestamps web api >"$generated_fixture"
|
||||
fixture="$generated_fixture"
|
||||
fi
|
||||
[ -f "$fixture" ] && [ ! -L "$fixture" ] || {
|
||||
echo "log fixture is missing" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if ! rows="$("$py" "$script_dir/classify_error_log.py" "$fixture" "$mode" "$now_text" "$error_threshold" "$fivexx_threshold" | tr -d '\r')"; then
|
||||
echo "could not classify logs" >&2
|
||||
exit 1
|
||||
fi
|
||||
stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
text=""
|
||||
if [ -n "$rows" ]; then
|
||||
while IFS=$'\t' read -r service route error_code count request_ids || [ -n "${service:-}" ]; do
|
||||
[ -n "${service:-}" ] || continue
|
||||
line="service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_ids:--} time=${stamp}"
|
||||
text="${text}${line}"$'\n'
|
||||
done <<<"$rows" || true
|
||||
elif [ "$mode" = "daily" ]; then
|
||||
text="service=all route=- errorCode=daily_summary count=0 requestId=- time=${stamp}"$'\n'
|
||||
fi
|
||||
|
||||
if [ -n "$text" ]; then
|
||||
post_monitor_alert "${text%$'\n'}"
|
||||
fi
|
||||
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe the public health URL, or the host disk. Install the health mode on
|
||||
# the Gitea runner, not on the production host. Install disk mode on the host.
|
||||
set -euo pipefail
|
||||
set +x
|
||||
umask 077
|
||||
|
||||
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=notify-feishu.sh
|
||||
source "$script_dir/notify-feishu.sh"
|
||||
|
||||
mode="health"
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--mode)
|
||||
mode="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--mode=*)
|
||||
mode="${1#--mode=}"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
case "$mode" in
|
||||
health|disk|all) ;;
|
||||
*)
|
||||
echo "unknown mode" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
env_file="${JYOTISHA_MONITOR_ENV:-/opt/jyotisha-production/.monitor.env}"
|
||||
load_monitor_env "$env_file"
|
||||
|
||||
health_url="${JYOTISHA_HEALTH_URL:-https://jyotisha.chat/api/health}"
|
||||
timeout="${JYOTISHA_HEALTH_TIMEOUT:-10}"
|
||||
[[ "$timeout" =~ ^[1-9][0-9]?$ ]] && (( timeout <= 30 )) || {
|
||||
echo "health timeout must be 1 to 30 seconds" >&2
|
||||
exit 2
|
||||
}
|
||||
if [ "$mode" != "disk" ]; then
|
||||
[[ "$health_url" == "https://jyotisha.chat/api/health" \
|
||||
|| "$health_url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/health$ \
|
||||
|| "$health_url" =~ ^http://localhost:[0-9]{1,5}/health$ ]] || {
|
||||
echo "unexpected health url" >&2
|
||||
exit 2
|
||||
}
|
||||
fi
|
||||
|
||||
state_file="$state_dir/monitor.state"
|
||||
consecutive_failures=0
|
||||
alert_open=0
|
||||
last_commit=""
|
||||
commit_alerted=""
|
||||
disk_alert_open=0
|
||||
|
||||
read_state() {
|
||||
local line value
|
||||
[ -f "$state_file" ] && [ ! -L "$state_file" ] || return 0
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
consecutive_failures=[0-9]*)
|
||||
consecutive_failures="${line#consecutive_failures=}"
|
||||
;;
|
||||
alert_open=[01])
|
||||
alert_open="${line#alert_open=}"
|
||||
;;
|
||||
last_commit=)
|
||||
last_commit=""
|
||||
;;
|
||||
last_commit=*)
|
||||
value="${line#last_commit=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && last_commit="$value"
|
||||
;;
|
||||
commit_alerted=)
|
||||
commit_alerted=""
|
||||
;;
|
||||
commit_alerted=*)
|
||||
value="${line#commit_alerted=}"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] && commit_alerted="$value"
|
||||
;;
|
||||
disk_alert_open=[01])
|
||||
disk_alert_open="${line#disk_alert_open=}"
|
||||
;;
|
||||
esac
|
||||
done <"$state_file"
|
||||
}
|
||||
|
||||
write_state() {
|
||||
local tmp
|
||||
tmp="$state_file.tmp.$$"
|
||||
printf 'consecutive_failures=%s\nalert_open=%s\nlast_commit=%s\ncommit_alerted=%s\ndisk_alert_open=%s\n' \
|
||||
"$consecutive_failures" "$alert_open" "$last_commit" "$commit_alerted" "$disk_alert_open" >"$tmp"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$tmp"
|
||||
fi
|
||||
mv "$tmp" "$state_file"
|
||||
}
|
||||
|
||||
stamp() {
|
||||
date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
local service="$1"
|
||||
local route="$2"
|
||||
local error_code="$3"
|
||||
local count="$4"
|
||||
local request_id="${5:--}"
|
||||
post_monitor_alert "service=${service} route=${route} errorCode=${error_code} count=${count} requestId=${request_id} time=$(stamp)"
|
||||
}
|
||||
|
||||
release_window_open() {
|
||||
local marker="$state_dir/release-window"
|
||||
local mtime now age
|
||||
[ -f "$marker" ] && [ ! -L "$marker" ] || return 1
|
||||
mtime="$(stat -c %Y "$marker" 2>/dev/null || stat -f %m "$marker")"
|
||||
now="$(date +%s)"
|
||||
age=$((now - mtime))
|
||||
[ "$age" -ge 0 ] && [ "$age" -le 2700 ]
|
||||
}
|
||||
|
||||
check_health() {
|
||||
local body_file http_code curl_status status_line commit_line status commit healthy py
|
||||
body_file="$(mktemp)"
|
||||
http_code="000"
|
||||
curl_status=0
|
||||
http_code="$(curl --silent --show-error --connect-timeout "$timeout" --max-time "$timeout" \
|
||||
-o "$body_file" -w '%{http_code}' "$health_url")" || curl_status=$?
|
||||
status=""
|
||||
commit=""
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ]; then
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
status_line="$("$py" - "$body_file" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
try:
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
except Exception:
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
if not isinstance(payload, dict):
|
||||
print("")
|
||||
print("")
|
||||
raise SystemExit(0)
|
||||
status = payload.get("status")
|
||||
deployment = payload.get("deployment")
|
||||
commit = ""
|
||||
if isinstance(deployment, dict):
|
||||
raw = deployment.get("gitCommit")
|
||||
if isinstance(raw, str):
|
||||
commit = raw
|
||||
print(status if isinstance(status, str) else "")
|
||||
print(commit)
|
||||
PY
|
||||
)"
|
||||
status="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==1 {print}')"
|
||||
commit="$(printf '%s\n' "$status_line" | tr -d '\r' | awk 'NR==2 {print}')"
|
||||
fi
|
||||
rm -f -- "$body_file"
|
||||
|
||||
healthy=0
|
||||
if [ "$curl_status" -eq 0 ] && [ "$http_code" = "200" ] && [ "$status" = "ok" ]; then
|
||||
healthy=1
|
||||
fi
|
||||
|
||||
if [ "$healthy" -eq 1 ]; then
|
||||
if [ "$alert_open" -eq 1 ]; then
|
||||
send_alert site /api/health recovered 0 -
|
||||
alert_open=0
|
||||
fi
|
||||
consecutive_failures=0
|
||||
if [[ "$commit" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
if [ -n "$last_commit" ] && [ "$commit" != "$last_commit" ] && [ "$commit_alerted" != "$commit" ]; then
|
||||
if ! release_window_open; then
|
||||
send_alert site /api/health commit_changed 1 -
|
||||
commit_alerted="$commit"
|
||||
fi
|
||||
fi
|
||||
last_commit="$commit"
|
||||
fi
|
||||
else
|
||||
consecutive_failures=$((consecutive_failures + 1))
|
||||
if [ "$consecutive_failures" -ge 3 ] && [ "$alert_open" -eq 0 ]; then
|
||||
send_alert site /api/health health_down "$consecutive_failures" -
|
||||
alert_open=1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
disk_percent() {
|
||||
df -Pk "${JYOTISHA_DISK_PATH:-/}" | awk 'NR==2 {gsub(/%/, "", $5); print $5}'
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
local percent
|
||||
percent="$(disk_percent)"
|
||||
[[ "$percent" =~ ^[0-9]+$ ]] && [ "$percent" -le 100 ] || {
|
||||
echo "disk usage is unreadable" >&2
|
||||
exit 1
|
||||
}
|
||||
if [ "$percent" -ge 80 ]; then
|
||||
if [ "$disk_alert_open" -eq 0 ]; then
|
||||
send_alert host - disk_high "$percent" -
|
||||
disk_alert_open=1
|
||||
fi
|
||||
elif [ "$disk_alert_open" -eq 1 ]; then
|
||||
send_alert host - recovered 0 -
|
||||
disk_alert_open=0
|
||||
fi
|
||||
}
|
||||
|
||||
read_state
|
||||
case "$mode" in
|
||||
health) check_health ;;
|
||||
disk) check_disk ;;
|
||||
all)
|
||||
check_health
|
||||
check_disk
|
||||
;;
|
||||
esac
|
||||
write_state
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shared alert sender. Source this file; do not execute it.
|
||||
# Outbound text is limited to service, route, errorCode, count, requestId, time.
|
||||
|
||||
ensure_private_dir() {
|
||||
local dir="$1"
|
||||
mkdir -p -- "$dir"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 700 "$dir"
|
||||
fi
|
||||
}
|
||||
|
||||
load_monitor_env() {
|
||||
local file="$1"
|
||||
local line key value mode
|
||||
[ -n "$file" ] || return 0
|
||||
[ -f "$file" ] && [ ! -L "$file" ] || return 0
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
mode="$(stat -c %a "$file")"
|
||||
[ "$mode" = "600" ] || {
|
||||
echo "monitor env must be mode 0600" >&2
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
case "$line" in
|
||||
""|\#*) continue ;;
|
||||
FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*)
|
||||
key="${line%%=*}"
|
||||
value="${line#*=}"
|
||||
case "$value" in
|
||||
*[$'\n\r`$']*)
|
||||
echo "monitor env value is unsafe" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
printf -v "$key" '%s' "$value"
|
||||
export "$key"
|
||||
;;
|
||||
esac
|
||||
done <"$file"
|
||||
}
|
||||
|
||||
feishu_sign() {
|
||||
local timestamp="$1"
|
||||
local secret="$2"
|
||||
local string_to_sign
|
||||
string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")"
|
||||
printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A
|
||||
}
|
||||
|
||||
webhook_allowed() {
|
||||
local url="$1"
|
||||
[[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0
|
||||
[[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
alert_line_allowed() {
|
||||
local line="$1"
|
||||
[[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
||||
}
|
||||
|
||||
post_monitor_alert() {
|
||||
local text="$1"
|
||||
local state_dir line timestamp sign payload_file
|
||||
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
||||
ensure_private_dir "$state_dir"
|
||||
while IFS= read -r line || [ -n "$line" ]; do
|
||||
[ -n "$line" ] || continue
|
||||
alert_line_allowed "$line" || {
|
||||
echo "alert text has a field that is not allowed" >&2
|
||||
exit 1
|
||||
}
|
||||
printf '%s\n' "$line" >>"$state_dir/alerts.log"
|
||||
done <<<"$text"
|
||||
|
||||
[ -n "${FEISHU_WEBHOOK_URL:-}" ] || return 0
|
||||
webhook_allowed "$FEISHU_WEBHOOK_URL" || {
|
||||
printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log"
|
||||
return 0
|
||||
}
|
||||
|
||||
payload_file="$(mktemp)"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
chmod 600 "$payload_file"
|
||||
fi
|
||||
if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then
|
||||
timestamp="$(date +%s)"
|
||||
sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")"
|
||||
FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \
|
||||
python_write_payload "$payload_file" signed
|
||||
else
|
||||
FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain
|
||||
fi
|
||||
curl --silent --show-error --connect-timeout 10 --max-time 10 \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'Connection: close' \
|
||||
--data-binary @"$payload_file" \
|
||||
"$FEISHU_WEBHOOK_URL" >/dev/null
|
||||
rm -f -- "$payload_file"
|
||||
}
|
||||
|
||||
python_write_payload() {
|
||||
local destination="$1"
|
||||
local mode="$2"
|
||||
local py
|
||||
export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN
|
||||
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
||||
py="$JYOTISHA_PYTHON"
|
||||
elif command -v python3 >/dev/null 2>&1; then
|
||||
py="python3"
|
||||
else
|
||||
py="python"
|
||||
fi
|
||||
"$py" - "$destination" "$mode" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
destination, mode = sys.argv[1], sys.argv[2]
|
||||
text = os.environ["FEISHU_ALERT_TEXT"]
|
||||
payload = {"msg_type": "text", "content": {"text": text}}
|
||||
if mode == "signed":
|
||||
payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"]
|
||||
payload["sign"] = os.environ["FEISHU_ALERT_SIGN"]
|
||||
with open(destination, "w", encoding="utf-8", newline="\n") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, separators=(",", ":"))
|
||||
PY
|
||||
}
|
||||
Reference in New Issue
Block a user