Expired daily files are pruned before a new backup, a verify older than 36 hours raises an alert, and the host monitors call sudo with the production env file. The Linux 0600 check is on the staging quick list. Co-Authored-By: Grok (agent loop)
171 lines
5.1 KiB
Bash
171 lines
5.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Shared alert sender. Source this file; do not execute it.
|
|
# Outbound text is limited to service, route, errorCode, count, requestId, time.
|
|
|
|
ensure_private_dir() {
|
|
local dir="$1"
|
|
mkdir -p -- "$dir"
|
|
if [ "$(uname -s)" = "Linux" ]; then
|
|
chmod 700 "$dir"
|
|
fi
|
|
}
|
|
|
|
load_monitor_env() {
|
|
local file="$1"
|
|
local line key value mode
|
|
[ -n "$file" ] || return 0
|
|
[ -f "$file" ] && [ ! -L "$file" ] || return 0
|
|
if [ "$(uname -s)" = "Linux" ]; then
|
|
mode="$(stat -c %a "$file")"
|
|
[ "$mode" = "600" ] || {
|
|
echo "monitor env must be mode 0600" >&2
|
|
exit 1
|
|
}
|
|
fi
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
case "$line" in
|
|
""|\#*) continue ;;
|
|
FEISHU_WEBHOOK_URL=*|FEISHU_SIGN_SECRET=*)
|
|
key="${line%%=*}"
|
|
value="${line#*=}"
|
|
case "$value" in
|
|
*[$'\n\r`$']*)
|
|
echo "monitor env value is unsafe" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
printf -v "$key" '%s' "$value"
|
|
export "$key"
|
|
;;
|
|
esac
|
|
done <"$file"
|
|
}
|
|
|
|
feishu_sign() {
|
|
local timestamp="$1"
|
|
local secret="$2"
|
|
local string_to_sign
|
|
string_to_sign="$(printf '%s\n%s' "$timestamp" "$secret")"
|
|
printf '' | openssl dgst -sha256 -hmac "$string_to_sign" -binary | openssl base64 -A
|
|
}
|
|
|
|
webhook_allowed() {
|
|
local url="$1"
|
|
[[ "$url" =~ ^https://open\.(feishu\.cn|larksuite\.com|larkoffice\.com)/open-apis/bot/v2/hook/[A-Za-z0-9-]+$ ]] && return 0
|
|
[[ "$url" =~ ^http://127\.0\.0\.1:[0-9]{1,5}/hook$ ]] && return 0
|
|
return 1
|
|
}
|
|
|
|
alert_line_allowed() {
|
|
local line="$1"
|
|
[[ "$line" =~ ^service=[a-z0-9_-]+[[:space:]]route=(/|/[A-Za-z0-9/_.-]+|-)[[:space:]]errorCode=[a-z][a-z0-9._-]*[[:space:]]count=[0-9]+[[:space:]]requestId=(-|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(,[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}){0,4})[[:space:]]time=[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]
|
|
}
|
|
|
|
post_monitor_alert() {
|
|
local text="$1"
|
|
local state_dir line timestamp sign payload_file response_file pending py curl_status code
|
|
state_dir="${JYOTISHA_MONITOR_STATE_DIR:-/var/lib/jyotisha-monitor}"
|
|
ensure_private_dir "$state_dir"
|
|
pending=""
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
[ -n "$line" ] || continue
|
|
alert_line_allowed "$line" || {
|
|
echo "alert text has a field that is not allowed" >&2
|
|
exit 1
|
|
}
|
|
pending="${pending}${line}"$'\n'
|
|
done <<<"$text"
|
|
[ -n "$pending" ] || return 0
|
|
pending="${pending%$'\n'}"
|
|
|
|
if [ -z "${FEISHU_WEBHOOK_URL:-}" ]; then
|
|
printf '%s\n' "$pending" >>"$state_dir/alerts.log"
|
|
return 0
|
|
fi
|
|
webhook_allowed "$FEISHU_WEBHOOK_URL" || {
|
|
printf '%s\n' "service=monitor route=- errorCode=webhook_rejected count=1 requestId=- time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"$state_dir/alerts.log"
|
|
return 0
|
|
}
|
|
|
|
payload_file="$(mktemp)"
|
|
response_file="$(mktemp)"
|
|
if [ "$(uname -s)" = "Linux" ]; then
|
|
chmod 600 "$payload_file" "$response_file"
|
|
fi
|
|
if [ -n "${FEISHU_SIGN_SECRET:-}" ]; then
|
|
timestamp="$(date +%s)"
|
|
sign="$(feishu_sign "$timestamp" "$FEISHU_SIGN_SECRET")"
|
|
FEISHU_ALERT_TEXT="$text" FEISHU_ALERT_TIMESTAMP="$timestamp" FEISHU_ALERT_SIGN="$sign" \
|
|
python_write_payload "$payload_file" signed
|
|
else
|
|
FEISHU_ALERT_TEXT="$text" python_write_payload "$payload_file" plain
|
|
fi
|
|
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
|
py="$JYOTISHA_PYTHON"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
py="python3"
|
|
else
|
|
py="python"
|
|
fi
|
|
curl_status=0
|
|
curl --silent --show-error --fail --connect-timeout 10 --max-time 10 \
|
|
-H 'Content-Type: application/json' \
|
|
-H 'Connection: close' \
|
|
--data-binary @"$payload_file" \
|
|
-o "$response_file" \
|
|
"$FEISHU_WEBHOOK_URL" || curl_status=$?
|
|
code=""
|
|
if [ "$curl_status" -eq 0 ]; then
|
|
code="$("$py" - "$response_file" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
try:
|
|
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
|
except Exception:
|
|
print("")
|
|
raise SystemExit(0)
|
|
if isinstance(payload, dict) and payload.get("code") == 0:
|
|
print("0")
|
|
else:
|
|
print("")
|
|
PY
|
|
)"
|
|
code="$(printf '%s' "$code" | tr -d '\r')"
|
|
fi
|
|
rm -f -- "$payload_file" "$response_file"
|
|
if [ "$curl_status" -ne 0 ] || [ "$code" != "0" ]; then
|
|
echo "feishu webhook was not accepted" >&2
|
|
return 1
|
|
fi
|
|
printf '%s\n' "$pending" >>"$state_dir/alerts.log"
|
|
}
|
|
|
|
python_write_payload() {
|
|
local destination="$1"
|
|
local mode="$2"
|
|
local py
|
|
export FEISHU_ALERT_TEXT FEISHU_ALERT_TIMESTAMP FEISHU_ALERT_SIGN
|
|
if [ -n "${JYOTISHA_PYTHON:-}" ]; then
|
|
py="$JYOTISHA_PYTHON"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
py="python3"
|
|
else
|
|
py="python"
|
|
fi
|
|
"$py" - "$destination" "$mode" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
destination, mode = sys.argv[1], sys.argv[2]
|
|
text = os.environ["FEISHU_ALERT_TEXT"]
|
|
payload = {"msg_type": "text", "content": {"text": text}}
|
|
if mode == "signed":
|
|
payload["timestamp"] = os.environ["FEISHU_ALERT_TIMESTAMP"]
|
|
payload["sign"] = os.environ["FEISHU_ALERT_SIGN"]
|
|
with open(destination, "w", encoding="utf-8", newline="\n") as handle:
|
|
json.dump(payload, handle, ensure_ascii=True, separators=(",", ":"))
|
|
PY
|
|
}
|