public-tools/EPICS2SMS.sh

#!/usr/bin/env bash

# ---------------------------------------------------------------------------
# EPICS monitor -> SMS/mail alerting
#
# Runs one or more independent monitors, each watching an EPICS PV:
#   - polls the PV at its own interval
#   - keeps a moving average over the last N samples ("N Mittel") so that a
#     single spurious reading cannot trigger an alarm
#   - raises an ALARM when the averaged value crosses a directional threshold
#     ("<" or ">")
#   - while in alarm, sends ONE message per poll, up to N Messages, then goes
#     silent until the value recovers
#   - resets the message counter once the averaged value returns to normal
#
# Multiple monitors may watch the SAME PV with different thresholds / timing;
# each runs as its own background loop with independent state.
#
# Mail/SMS transport is delegated to smtp_send.py (direct SMTP to smtp.psi.ch).
# IMPORTANT: From address must be a registered sender (e.g. cSAXS@psi.ch).
# ---------------------------------------------------------------------------

# --- Configuration ----------------------------------------------------------
# All site-specific settings (recipients, sender, PVs/thresholds, timing) live
# in a separate, UNTRACKED config file, so that "git pull" never touches them.
# Copy the template once and edit it:
#
#   cp epics2sms.conf.example epics2sms.conf
#   $EDITOR epics2sms.conf
#
# The config file is sourced as bash and may set: RECIPIENTS (array), FROM_ADDR,
# ALERT_SUBJECT, CA_TIMEOUT and MONITORS (array). See epics2sms.conf.example for
# the field format. Its path defaults to epics2sms.conf next to this script;
# override with the EPICS2SMS_CONF environment variable or the first argument.
# ----------------------------------------------------------------------------

# Resolve paths relative to this script, not the current working directory.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
PYTHON_SENDER="${SCRIPT_DIR}/smtp_send.py"

# Defaults; the config file may override any of these.
ALERT_SUBJECT="EPICS alert"
FROM_ADDR="cSAXS@psi.ch"
CA_TIMEOUT=5            # caget Channel Access timeout (s); keep below shortest poll
RECIPIENTS=()
MONITORS=()

# Config file location: CLI arg > EPICS2SMS_CONF env var > default beside script.
CONF_FILE="${EPICS2SMS_CONF:-${SCRIPT_DIR}/epics2sms.conf}"
[[ -n "${1:-}" ]] && CONF_FILE="$1"

# --- helpers ----------------------------------------------------------------

log() {
  printf '[%s] %s\n' "$(date '+%F %T')" "$*"
}

# Per-monitor log line, tagged with the monitor label.
mlog() {
  local tag="$1"; shift
  printf '[%s] [%s] %s\n' "$(date '+%F %T')" "$tag" "$*"
}

require_command() {
  local cmd="$1"
  command -v "$cmd" >/dev/null 2>&1 || {
    log "ERROR: required command not found: $cmd"
    exit 2
  }
}

is_numeric() {
  # Accepts integers, decimals, and scientific notation.
  # Examples: 1, -1, 1.0, .5, 1e-3, -2.3E+4
  local s="$1"
  [[ "$s" =~ ^-?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?$ ]]
}

send_alert() {
  local body="$1"
  local subject="${2:-$ALERT_SUBJECT}"

  local rc=0
  local to
  for to in "${RECIPIENTS[@]}"; do
    # smtp_send.py expects: --to (repeatable), optional flags, then message.
    if ! python3 "$PYTHON_SENDER" \
      --to "$to" \
      --subject "$subject" \
      --from-addr "$FROM_ADDR" \
      "$body"; then
      rc=1
    fi
  done

  return "$rc"
}

# --- monitor loop -----------------------------------------------------------
# Runs as a background subshell so its state (window, counters) is isolated.
monitor_loop() {
  local pv="$1" dir="$2" thresh="$3" poll="$4" apoll="$5" n_avg="$6" n_msg="$7" msg_delay="$8" text="$9"
  local tag="$pv $dir$thresh"

  local -a window=()
  local msg_count=0
  local in_alarm=0
  local last_msg_ts=0
  local value status avg alarm body now

  mlog "$tag" "monitor start: dir=$dir thresh=$thresh poll=${poll}s alarm_poll=${apoll}s n_avg=$n_avg n_msg=$n_msg msg_delay=${msg_delay}s"

  while true; do
    value="$(caget -w "$CA_TIMEOUT" -noname -nounit "$pv" 2>/dev/null)"
    status=$?

    # Transient read failures must not poison the average -> skip the sample.
    if [[ $status -ne 0 ]] || ! is_numeric "$value"; then
      mlog "$tag" "WARNING: unreadable/non-numeric value ('$value', caget exit $status) - sample skipped"
      (( in_alarm )) && sleep "$apoll" || sleep "$poll"
      continue
    fi

    # Push into the rolling window, dropping the oldest sample past n_avg.
    window+=("$value")
    if (( ${#window[@]} > n_avg )); then
      window=("${window[@]:1}")
    fi

    # Need a full window before any alarm may fire.
    if (( ${#window[@]} < n_avg )); then
      sleep "$poll"
      continue
    fi

    avg="$(printf '%s\n' "${window[@]}" | awk '{s+=$1} END {printf "%.6g", s/NR}')"

    alarm=0
    if [[ "$dir" == "lt" ]]; then
      awk -v a="$avg" -v t="$thresh" 'BEGIN{exit !(a < t)}' && alarm=1
    else
      awk -v a="$avg" -v t="$thresh" 'BEGIN{exit !(a > t)}' && alarm=1
    fi

    if (( alarm )); then
      if (( ! in_alarm )); then
        in_alarm=1
        msg_count=0
        last_msg_ts=0   # 0 => first message of the episode goes out immediately
        mlog "$tag" "ALARM entered (avg=$avg, last=$value)"
      fi

      # Send at most n_msg messages, and no more often than every msg_delay
      # seconds even though polling continues at the faster alarm rate.
      now="$(date +%s)"
      if (( msg_count < n_msg && now - last_msg_ts >= msg_delay )); then
        body="$text (avg=$avg, last=$value) $(date '+%F %T')"
        if send_alert "$body" "$ALERT_SUBJECT"; then
          (( msg_count++ ))
          last_msg_ts="$now"
          mlog "$tag" "SMS $msg_count/$n_msg sent (avg=$avg)"
          if (( msg_count == n_msg )); then
            mlog "$tag" "message cap reached ($n_msg) - silent until recovery"
          fi
        else
          mlog "$tag" "ERROR: SMS send failed; will retry next poll"
        fi
      fi
    else
      if (( in_alarm )); then
        mlog "$tag" "recovered to normal (avg=$avg) after $msg_count message(s)"
        # Recovery notification: only if we actually alerted during this
        # episode; sent once, immediately (not throttled/capped).
        if (( msg_count > 0 )); then
          body="RESOLVED: $text - now avg=$avg (last=$value) $(date '+%F %T')"
          if send_alert "$body" "EPICS resolved"; then
            mlog "$tag" "resolve notification sent"
          else
            mlog "$tag" "ERROR: resolve notification failed to send"
          fi
        fi
      fi
      in_alarm=0
      msg_count=0
      last_msg_ts=0
    fi

    # Poll at the alarm rate while in alarm (keeps the average and recovery
    # check fresh); message spacing is governed separately by msg_delay.
    if (( in_alarm )); then
      sleep "$apoll"
    else
      sleep "$poll"
    fi
  done
}

# --- startup checks ---------------------------------------------------------

require_command caget
require_command python3

if [[ ! -f "$PYTHON_SENDER" ]]; then
  log "ERROR: Python sender script not found: $PYTHON_SENDER"
  exit 2
fi

# --- load site configuration ------------------------------------------------

if [[ ! -f "$CONF_FILE" ]]; then
  log "ERROR: config file not found: $CONF_FILE"
  log "       Copy the template and edit it, e.g.:"
  log "         cp '${SCRIPT_DIR}/epics2sms.conf.example' '$CONF_FILE'"
  exit 2
fi

# shellcheck source=/dev/null
source "$CONF_FILE"

if (( ${#RECIPIENTS[@]} == 0 )); then
  log "ERROR: no RECIPIENTS defined in $CONF_FILE"
  exit 2
fi
if (( ${#MONITORS[@]} == 0 )); then
  log "ERROR: no MONITORS defined in $CONF_FILE"
  exit 2
fi

log "loaded config: $CONF_FILE (${#MONITORS[@]} monitor(s), ${#RECIPIENTS[@]} recipient(s))"

# --- launch monitors --------------------------------------------------------

pids=()
summaries=()
cleaned=0

cleanup() {
  (( cleaned )) && return
  cleaned=1
  log "shutting down; stopping ${#pids[@]} monitor(s)"
  local p
  for p in "${pids[@]}"; do
    kill "$p" 2>/dev/null
  done
  wait 2>/dev/null
}
trap cleanup EXIT INT TERM

for entry in "${MONITORS[@]}"; do
  IFS='|' read -r pv dir thresh poll apoll n_avg n_msg msg_delay text <<< "$entry"

  # Basic validation; skip malformed entries rather than aborting the whole run.
  if [[ -z "$pv" || -z "$dir" || -z "$thresh" || -z "$poll" || -z "$apoll" || -z "$n_avg" || -z "$n_msg" || -z "$msg_delay" || -z "$text" ]]; then
    log "ERROR: malformed monitor entry (missing field): '$entry' - skipped"
    continue
  fi
  if [[ "$dir" != "lt" && "$dir" != "gt" ]]; then
    log "ERROR: monitor '$pv': invalid dir '$dir' (expected lt|gt) - skipped"
    continue
  fi
  if ! is_numeric "$thresh"; then
    log "ERROR: monitor '$pv': non-numeric threshold '$thresh' - skipped"
    continue
  fi
  if ! [[ "$poll" =~ ^[0-9]+$ && "$apoll" =~ ^[0-9]+$ && "$n_avg" =~ ^[0-9]+$ && "$n_msg" =~ ^[0-9]+$ && "$msg_delay" =~ ^[0-9]+$ ]] \
     || (( poll < 1 || apoll < 1 || n_avg < 1 || n_msg < 1 || msg_delay < 1 )); then
    log "ERROR: monitor '$pv': poll/alarm_poll/n_avg/n_msg/msg_delay must be positive integers - skipped"
    continue
  fi

  monitor_loop "$pv" "$dir" "$thresh" "$poll" "$apoll" "$n_avg" "$n_msg" "$msg_delay" "$text" &
  pids+=("$!")
  # Brief per-monitor line for the startup notification.
  summaries+=("$text (poll ${poll}s/alarm ${apoll}s, avg ${n_avg}, max ${n_msg} msg every ${msg_delay}s)")
done

if (( ${#pids[@]} == 0 )); then
  log "ERROR: no valid monitors to run"
  exit 2
fi

# Send a brief startup notification so recipients know the monitor is running
# and what it watches. Best-effort: a failure here must not stop monitoring.
startup_body="EPICS2SMS started $(date '+%F %T') by $(id -un) on $(hostname -s), ${#pids[@]} monitor(s):"
for line in "${summaries[@]}"; do
  startup_body+=$'\n'"- $line"
done
if send_alert "$startup_body" "EPICS2SMS started"; then
  log "startup notification sent"
else
  log "WARNING: startup notification failed to send"
fi

log "started ${#pids[@]} monitor(s); polling..."
wait