epics2sms: throttle alarm messages (msg_delay) and send a recovery message
Add a per-monitor msg_delay_s (default 300s): polling continues at the alarm rate to keep the average and recovery check fresh, but alarm messages are throttled to at most one per msg_delay_s. The first message of an episode is still sent immediately. Also send one "RESOLVED" notification when a monitor returns to normal (immediate, not throttled or capped). Config format: pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+35
-18
@@ -39,22 +39,24 @@ CA_TIMEOUT=5
|
||||
# --- Monitor definitions ----------------------------------------------------
|
||||
# One entry per monitor, pipe-separated:
|
||||
#
|
||||
# pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text
|
||||
# pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text
|
||||
#
|
||||
# pv EPICS PV name
|
||||
# dir alarm direction: "lt" (avg < threshold) or "gt" (avg > threshold)
|
||||
# threshold numeric threshold
|
||||
# poll_s seconds between polls while in normal state
|
||||
# alarm_poll_s seconds between polls while in alarm state; since one message
|
||||
# is sent per alarm poll, this also sets the minimum interval
|
||||
# between messages
|
||||
# alarm_poll_s seconds between polls while in alarm state (how often the
|
||||
# average is refreshed and recovery is checked)
|
||||
# n_avg number of samples to average before an alarm can fire
|
||||
# n_msg max messages to send per alarm episode, then stay silent
|
||||
# msg_delay_s minimum seconds between two alarm messages; polling keeps
|
||||
# running faster, but messages are throttled to this spacing
|
||||
# (the first message of an episode is sent immediately)
|
||||
# text message text (also becomes the start of the SMS body)
|
||||
#
|
||||
MONITORS=(
|
||||
"X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|X12SA-OP-CC:L1Level_MON <55%"
|
||||
"X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|X12SA-OP-CC:L1Level_MON >75%"
|
||||
"X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|300|X12SA-OP-CC:L1Level_MON <55%"
|
||||
"X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|300|X12SA-OP-CC:L1Level_MON >75%"
|
||||
)
|
||||
|
||||
# Resolve smtp_send.py relative to this script, not the current working directory.
|
||||
@@ -111,15 +113,16 @@ send_alert() {
|
||||
# --- monitor loop -----------------------------------------------------------
|
||||
# Runs as a background subshell so its state (window, counters) is isolated.
|
||||
monitor_loop() {
|
||||
local pv="$1" dir="$2" thresh="$3" poll="$4" apoll="$5" n_avg="$6" n_msg="$7" text="$8"
|
||||
local pv="$1" dir="$2" thresh="$3" poll="$4" apoll="$5" n_avg="$6" n_msg="$7" msg_delay="$8" text="$9"
|
||||
local tag="$pv $dir$thresh"
|
||||
|
||||
local -a window=()
|
||||
local msg_count=0
|
||||
local in_alarm=0
|
||||
local value status avg alarm body
|
||||
local last_msg_ts=0
|
||||
local value status avg alarm body now
|
||||
|
||||
mlog "$tag" "monitor start: dir=$dir thresh=$thresh poll=${poll}s alarm_poll=${apoll}s n_avg=$n_avg n_msg=$n_msg"
|
||||
mlog "$tag" "monitor start: dir=$dir thresh=$thresh poll=${poll}s alarm_poll=${apoll}s n_avg=$n_avg n_msg=$n_msg msg_delay=${msg_delay}s"
|
||||
|
||||
while true; do
|
||||
value="$(caget -w "$CA_TIMEOUT" -noname -nounit "$pv" 2>/dev/null)"
|
||||
@@ -157,13 +160,18 @@ monitor_loop() {
|
||||
if (( ! in_alarm )); then
|
||||
in_alarm=1
|
||||
msg_count=0
|
||||
last_msg_ts=0 # 0 => first message of the episode goes out immediately
|
||||
mlog "$tag" "ALARM entered (avg=$avg, last=$value)"
|
||||
fi
|
||||
|
||||
if (( msg_count < n_msg )); then
|
||||
# Send at most n_msg messages, and no more often than every msg_delay
|
||||
# seconds even though polling continues at the faster alarm rate.
|
||||
now="$(date +%s)"
|
||||
if (( msg_count < n_msg && now - last_msg_ts >= msg_delay )); then
|
||||
body="$text (avg=$avg, last=$value) $(date '+%F %T')"
|
||||
if send_alert "$body" "$ALERT_SUBJECT"; then
|
||||
(( msg_count++ ))
|
||||
last_msg_ts="$now"
|
||||
mlog "$tag" "SMS $msg_count/$n_msg sent (avg=$avg)"
|
||||
if (( msg_count == n_msg )); then
|
||||
mlog "$tag" "message cap reached ($n_msg) - silent until recovery"
|
||||
@@ -175,12 +183,21 @@ monitor_loop() {
|
||||
else
|
||||
if (( in_alarm )); then
|
||||
mlog "$tag" "recovered to normal (avg=$avg) after $msg_count message(s)"
|
||||
# Recovery notification: sent once, immediately (not throttled/capped).
|
||||
body="RESOLVED: $text - now avg=$avg (last=$value) $(date '+%F %T')"
|
||||
if send_alert "$body" "EPICS resolved"; then
|
||||
mlog "$tag" "resolve notification sent"
|
||||
else
|
||||
mlog "$tag" "ERROR: resolve notification failed to send"
|
||||
fi
|
||||
fi
|
||||
in_alarm=0
|
||||
msg_count=0
|
||||
last_msg_ts=0
|
||||
fi
|
||||
|
||||
# Poll faster (or slower) during an alarm; this also spaces the messages.
|
||||
# Poll at the alarm rate while in alarm (keeps the average and recovery
|
||||
# check fresh); message spacing is governed separately by msg_delay.
|
||||
if (( in_alarm )); then
|
||||
sleep "$apoll"
|
||||
else
|
||||
@@ -218,10 +235,10 @@ cleanup() {
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
for entry in "${MONITORS[@]}"; do
|
||||
IFS='|' read -r pv dir thresh poll apoll n_avg n_msg text <<< "$entry"
|
||||
IFS='|' read -r pv dir thresh poll apoll n_avg n_msg msg_delay text <<< "$entry"
|
||||
|
||||
# Basic validation; skip malformed entries rather than aborting the whole run.
|
||||
if [[ -z "$pv" || -z "$dir" || -z "$thresh" || -z "$poll" || -z "$apoll" || -z "$n_avg" || -z "$n_msg" || -z "$text" ]]; then
|
||||
if [[ -z "$pv" || -z "$dir" || -z "$thresh" || -z "$poll" || -z "$apoll" || -z "$n_avg" || -z "$n_msg" || -z "$msg_delay" || -z "$text" ]]; then
|
||||
log "ERROR: malformed monitor entry (missing field): '$entry' - skipped"
|
||||
continue
|
||||
fi
|
||||
@@ -233,16 +250,16 @@ for entry in "${MONITORS[@]}"; do
|
||||
log "ERROR: monitor '$pv': non-numeric threshold '$thresh' - skipped"
|
||||
continue
|
||||
fi
|
||||
if ! [[ "$poll" =~ ^[0-9]+$ && "$apoll" =~ ^[0-9]+$ && "$n_avg" =~ ^[0-9]+$ && "$n_msg" =~ ^[0-9]+$ ]] \
|
||||
|| (( poll < 1 || apoll < 1 || n_avg < 1 || n_msg < 1 )); then
|
||||
log "ERROR: monitor '$pv': poll/alarm_poll/n_avg/n_msg must be positive integers - skipped"
|
||||
if ! [[ "$poll" =~ ^[0-9]+$ && "$apoll" =~ ^[0-9]+$ && "$n_avg" =~ ^[0-9]+$ && "$n_msg" =~ ^[0-9]+$ && "$msg_delay" =~ ^[0-9]+$ ]] \
|
||||
|| (( poll < 1 || apoll < 1 || n_avg < 1 || n_msg < 1 || msg_delay < 1 )); then
|
||||
log "ERROR: monitor '$pv': poll/alarm_poll/n_avg/n_msg/msg_delay must be positive integers - skipped"
|
||||
continue
|
||||
fi
|
||||
|
||||
monitor_loop "$pv" "$dir" "$thresh" "$poll" "$apoll" "$n_avg" "$n_msg" "$text" &
|
||||
monitor_loop "$pv" "$dir" "$thresh" "$poll" "$apoll" "$n_avg" "$n_msg" "$msg_delay" "$text" &
|
||||
pids+=("$!")
|
||||
# Brief per-monitor line for the startup notification.
|
||||
summaries+=("$text (poll ${poll}s/alarm ${apoll}s, avg ${n_avg}, max ${n_msg} msg)")
|
||||
summaries+=("$text (poll ${poll}s/alarm ${apoll}s, avg ${n_avg}, max ${n_msg} msg every ${msg_delay}s)")
|
||||
done
|
||||
|
||||
if (( ${#pids[@]} == 0 )); then
|
||||
|
||||
@@ -22,7 +22,7 @@ system/application sender.
|
||||
Each monitor is one pipe-separated entry in the `MONITORS` array:
|
||||
|
||||
```
|
||||
pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text
|
||||
pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text
|
||||
```
|
||||
|
||||
| Field | Meaning |
|
||||
@@ -31,9 +31,10 @@ pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text
|
||||
| `dir` | alarm direction: `lt` (avg `<` threshold) or `gt` (avg `>` threshold) |
|
||||
| `threshold` | numeric threshold |
|
||||
| `poll_s` | seconds between polls while in normal state |
|
||||
| `alarm_poll_s` | seconds between polls while in alarm; since one message is sent per alarm poll, this also sets the minimum interval between messages |
|
||||
| `alarm_poll_s` | seconds between polls while in alarm (how often the average is refreshed and recovery is checked) |
|
||||
| `n_avg` | samples averaged before an alarm can fire ("N Mittel") |
|
||||
| `n_msg` | max messages per alarm episode, then silent ("N Messages") |
|
||||
| `msg_delay_s` | minimum seconds between two alarm messages; polling keeps running faster, but messages are throttled to this spacing (the first message of an episode goes out immediately) |
|
||||
| `text` | message text, also the start of the SMS body |
|
||||
|
||||
Multiple monitors may watch the **same** PV with different thresholds and
|
||||
@@ -43,8 +44,8 @@ Example (current config):
|
||||
|
||||
```
|
||||
MONITORS=(
|
||||
"X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|X12SA-OP-CC:L1Level_MON <55%"
|
||||
"X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|X12SA-OP-CC:L1Level_MON >75%"
|
||||
"X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|300|X12SA-OP-CC:L1Level_MON <55%"
|
||||
"X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|300|X12SA-OP-CC:L1Level_MON >75%"
|
||||
)
|
||||
```
|
||||
|
||||
@@ -59,10 +60,11 @@ Behaviour of one monitor:
|
||||
in alarm
|
||||
- keeps a moving average of the last `n_avg` samples; an alarm can only fire
|
||||
once the window is full, so a single spurious reading cannot trigger it
|
||||
- while the average is in alarm, sends **one** message per alarm poll, up to
|
||||
`n_msg` messages, then stays silent — so `alarm_poll_s` governs the minimum
|
||||
spacing between messages
|
||||
- once the average returns to normal, the message counter resets
|
||||
- while the average is in alarm, sends the first message immediately, then up
|
||||
to `n_msg` messages total, spaced at least `msg_delay_s` apart, then stays
|
||||
silent
|
||||
- once the average returns to normal, it sends **one** recovery ("RESOLVED")
|
||||
message and resets the message counter
|
||||
- unreadable / non-numeric reads are logged and skipped (they do not enter the
|
||||
average and never trigger an alarm)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user