From 9aa5e463f89ea5f674b596f11110c724b82f4b87 Mon Sep 17 00:00:00 2001 From: menzel Date: Wed, 15 Jul 2026 12:35:53 +0200 Subject: [PATCH] epics2sms: throttle alarm messages (msg_delay) and send a recovery message Add a per-monitor msg_delay_s (default 300s): polling continues at the alarm rate to keep the average and recovery check fresh, but alarm messages are throttled to at most one per msg_delay_s. The first message of an episode is still sent immediately. Also send one "RESOLVED" notification when a monitor returns to normal (immediate, not throttled or capped). Config format: pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text Co-Authored-By: Claude Opus 4.8 (1M context) --- EPICS2SMS.sh | 53 ++++++++++++++++++++++++++++++++++------------------ README.md | 18 ++++++++++-------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/EPICS2SMS.sh b/EPICS2SMS.sh index f94f879..aa4fece 100755 --- a/EPICS2SMS.sh +++ b/EPICS2SMS.sh @@ -39,22 +39,24 @@ CA_TIMEOUT=5 # --- Monitor definitions ---------------------------------------------------- # One entry per monitor, pipe-separated: # -# pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text +# pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text # # pv EPICS PV name # dir alarm direction: "lt" (avg < threshold) or "gt" (avg > threshold) # threshold numeric threshold # poll_s seconds between polls while in normal state -# alarm_poll_s seconds between polls while in alarm state; since one message -# is sent per alarm poll, this also sets the minimum interval -# between messages +# alarm_poll_s seconds between polls while in alarm state (how often the +# average is refreshed and recovery is checked) # n_avg number of samples to average before an alarm can fire # n_msg max messages to send per alarm episode, then stay silent +# msg_delay_s minimum seconds between two alarm messages; polling keeps +# running faster, but messages are throttled to this spacing +# (the first message of an episode is sent immediately) # text message text (also becomes the start of the SMS body) # MONITORS=( - "X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|X12SA-OP-CC:L1Level_MON <55%" - "X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|X12SA-OP-CC:L1Level_MON >75%" + "X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|300|X12SA-OP-CC:L1Level_MON <55%" + "X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|300|X12SA-OP-CC:L1Level_MON >75%" ) # Resolve smtp_send.py relative to this script, not the current working directory. @@ -111,15 +113,16 @@ send_alert() { # --- monitor loop ----------------------------------------------------------- # Runs as a background subshell so its state (window, counters) is isolated. monitor_loop() { - local pv="$1" dir="$2" thresh="$3" poll="$4" apoll="$5" n_avg="$6" n_msg="$7" text="$8" + local pv="$1" dir="$2" thresh="$3" poll="$4" apoll="$5" n_avg="$6" n_msg="$7" msg_delay="$8" text="$9" local tag="$pv $dir$thresh" local -a window=() local msg_count=0 local in_alarm=0 - local value status avg alarm body + local last_msg_ts=0 + local value status avg alarm body now - mlog "$tag" "monitor start: dir=$dir thresh=$thresh poll=${poll}s alarm_poll=${apoll}s n_avg=$n_avg n_msg=$n_msg" + mlog "$tag" "monitor start: dir=$dir thresh=$thresh poll=${poll}s alarm_poll=${apoll}s n_avg=$n_avg n_msg=$n_msg msg_delay=${msg_delay}s" while true; do value="$(caget -w "$CA_TIMEOUT" -noname -nounit "$pv" 2>/dev/null)" @@ -157,13 +160,18 @@ monitor_loop() { if (( ! in_alarm )); then in_alarm=1 msg_count=0 + last_msg_ts=0 # 0 => first message of the episode goes out immediately mlog "$tag" "ALARM entered (avg=$avg, last=$value)" fi - if (( msg_count < n_msg )); then + # Send at most n_msg messages, and no more often than every msg_delay + # seconds even though polling continues at the faster alarm rate. + now="$(date +%s)" + if (( msg_count < n_msg && now - last_msg_ts >= msg_delay )); then body="$text (avg=$avg, last=$value) $(date '+%F %T')" if send_alert "$body" "$ALERT_SUBJECT"; then (( msg_count++ )) + last_msg_ts="$now" mlog "$tag" "SMS $msg_count/$n_msg sent (avg=$avg)" if (( msg_count == n_msg )); then mlog "$tag" "message cap reached ($n_msg) - silent until recovery" @@ -175,12 +183,21 @@ monitor_loop() { else if (( in_alarm )); then mlog "$tag" "recovered to normal (avg=$avg) after $msg_count message(s)" + # Recovery notification: sent once, immediately (not throttled/capped). + body="RESOLVED: $text - now avg=$avg (last=$value) $(date '+%F %T')" + if send_alert "$body" "EPICS resolved"; then + mlog "$tag" "resolve notification sent" + else + mlog "$tag" "ERROR: resolve notification failed to send" + fi fi in_alarm=0 msg_count=0 + last_msg_ts=0 fi - # Poll faster (or slower) during an alarm; this also spaces the messages. + # Poll at the alarm rate while in alarm (keeps the average and recovery + # check fresh); message spacing is governed separately by msg_delay. if (( in_alarm )); then sleep "$apoll" else @@ -218,10 +235,10 @@ cleanup() { trap cleanup EXIT INT TERM for entry in "${MONITORS[@]}"; do - IFS='|' read -r pv dir thresh poll apoll n_avg n_msg text <<< "$entry" + IFS='|' read -r pv dir thresh poll apoll n_avg n_msg msg_delay text <<< "$entry" # Basic validation; skip malformed entries rather than aborting the whole run. - if [[ -z "$pv" || -z "$dir" || -z "$thresh" || -z "$poll" || -z "$apoll" || -z "$n_avg" || -z "$n_msg" || -z "$text" ]]; then + if [[ -z "$pv" || -z "$dir" || -z "$thresh" || -z "$poll" || -z "$apoll" || -z "$n_avg" || -z "$n_msg" || -z "$msg_delay" || -z "$text" ]]; then log "ERROR: malformed monitor entry (missing field): '$entry' - skipped" continue fi @@ -233,16 +250,16 @@ for entry in "${MONITORS[@]}"; do log "ERROR: monitor '$pv': non-numeric threshold '$thresh' - skipped" continue fi - if ! [[ "$poll" =~ ^[0-9]+$ && "$apoll" =~ ^[0-9]+$ && "$n_avg" =~ ^[0-9]+$ && "$n_msg" =~ ^[0-9]+$ ]] \ - || (( poll < 1 || apoll < 1 || n_avg < 1 || n_msg < 1 )); then - log "ERROR: monitor '$pv': poll/alarm_poll/n_avg/n_msg must be positive integers - skipped" + if ! [[ "$poll" =~ ^[0-9]+$ && "$apoll" =~ ^[0-9]+$ && "$n_avg" =~ ^[0-9]+$ && "$n_msg" =~ ^[0-9]+$ && "$msg_delay" =~ ^[0-9]+$ ]] \ + || (( poll < 1 || apoll < 1 || n_avg < 1 || n_msg < 1 || msg_delay < 1 )); then + log "ERROR: monitor '$pv': poll/alarm_poll/n_avg/n_msg/msg_delay must be positive integers - skipped" continue fi - monitor_loop "$pv" "$dir" "$thresh" "$poll" "$apoll" "$n_avg" "$n_msg" "$text" & + monitor_loop "$pv" "$dir" "$thresh" "$poll" "$apoll" "$n_avg" "$n_msg" "$msg_delay" "$text" & pids+=("$!") # Brief per-monitor line for the startup notification. - summaries+=("$text (poll ${poll}s/alarm ${apoll}s, avg ${n_avg}, max ${n_msg} msg)") + summaries+=("$text (poll ${poll}s/alarm ${apoll}s, avg ${n_avg}, max ${n_msg} msg every ${msg_delay}s)") done if (( ${#pids[@]} == 0 )); then diff --git a/README.md b/README.md index 7259cc5..6107551 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ system/application sender. Each monitor is one pipe-separated entry in the `MONITORS` array: ``` -pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text +pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | msg_delay_s | text ``` | Field | Meaning | @@ -31,9 +31,10 @@ pv | dir | threshold | poll_s | alarm_poll_s | n_avg | n_msg | text | `dir` | alarm direction: `lt` (avg `<` threshold) or `gt` (avg `>` threshold) | | `threshold` | numeric threshold | | `poll_s` | seconds between polls while in normal state | -| `alarm_poll_s` | seconds between polls while in alarm; since one message is sent per alarm poll, this also sets the minimum interval between messages | +| `alarm_poll_s` | seconds between polls while in alarm (how often the average is refreshed and recovery is checked) | | `n_avg` | samples averaged before an alarm can fire ("N Mittel") | | `n_msg` | max messages per alarm episode, then silent ("N Messages") | +| `msg_delay_s` | minimum seconds between two alarm messages; polling keeps running faster, but messages are throttled to this spacing (the first message of an episode goes out immediately) | | `text` | message text, also the start of the SMS body | Multiple monitors may watch the **same** PV with different thresholds and @@ -43,8 +44,8 @@ Example (current config): ``` MONITORS=( - "X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|X12SA-OP-CC:L1Level_MON <55%" - "X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|X12SA-OP-CC:L1Level_MON >75%" + "X12SA-OP-CC:L1Level_MON|lt|55|60|60|4|10|300|X12SA-OP-CC:L1Level_MON <55%" + "X12SA-OP-CC:L1Level_MON|gt|75|10|10|4|10|300|X12SA-OP-CC:L1Level_MON >75%" ) ``` @@ -59,10 +60,11 @@ Behaviour of one monitor: in alarm - keeps a moving average of the last `n_avg` samples; an alarm can only fire once the window is full, so a single spurious reading cannot trigger it -- while the average is in alarm, sends **one** message per alarm poll, up to - `n_msg` messages, then stays silent — so `alarm_poll_s` governs the minimum - spacing between messages -- once the average returns to normal, the message counter resets +- while the average is in alarm, sends the first message immediately, then up + to `n_msg` messages total, spaced at least `msg_delay_s` apart, then stays + silent +- once the average returns to normal, it sends **one** recovery ("RESOLVED") + message and resets the message counter - unreadable / non-numeric reads are logged and skipped (they do not enter the average and never trigger an alarm)