Ops hardening: lead re-import, SMS health-watch, ambient office murmur
- scripts/reimport_leads.py drains appointment_requests.jsonl into Odoo (fallback leads were captured but invisible to staff forever); server.py now warns at startup when leads are pending. - scripts/healthwatch.sh (cron, every minute): after 3 consecutive /health failures sends a Twilio SMS to ALERT_SMS_TO (30-min cooldown) + a recovery SMS. systemd handles restarts; this makes a human aware when that isn't enough. Tested end-to-end (DOWN + RECOVERED SMS delivered). - Ambient office murmur (assets/office_ambience.wav, generated by scripts/make_ambience.py from low-passed unintelligible Kokoro babble + room tone) mixed continuously into outbound audio via pipecat SoundfileMixer, so callers hear a live office instead of dead silence while the agent thinks. AMBIENT_VOLUME/AMBIENT_ENABLED to tune/disable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
67
scripts/healthwatch.sh
Executable file
67
scripts/healthwatch.sh
Executable file
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# Health watch for the AVC phone agent. Run from cron every minute:
|
||||
# * * * * * /home/tocmo0nlord/avc-phone/scripts/healthwatch.sh
|
||||
#
|
||||
# Curls /health; after FAIL_THRESHOLD consecutive failures it sends ONE Twilio SMS to
|
||||
# ALERT_SMS_TO (re-alerts at most every ALERT_COOLDOWN_MIN), and sends a recovery SMS
|
||||
# when the service comes back. systemd already auto-restarts the process — this exists
|
||||
# so a human finds out when restarts aren't enough (crash loop, GPU wedged, box issues).
|
||||
# State lives in $STATE_DIR; activity is appended to healthwatch.log (gitignored).
|
||||
set -u
|
||||
|
||||
PROJ="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
set -a; . "$PROJ/.env" 2>/dev/null; set +a
|
||||
|
||||
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8200/health}"
|
||||
FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"
|
||||
ALERT_COOLDOWN_MIN="${ALERT_COOLDOWN_MIN:-30}"
|
||||
STATE_DIR="${STATE_DIR:-$HOME/.local/state/avc-healthwatch}"
|
||||
LOG="$PROJ/healthwatch.log"
|
||||
mkdir -p "$STATE_DIR"
|
||||
FAILS_F="$STATE_DIR/consecutive_fails"
|
||||
ALERTED_F="$STATE_DIR/alerted_at_epoch"
|
||||
|
||||
log() { echo "$(date '+%F %T') $*" >> "$LOG"; }
|
||||
|
||||
send_sms() {
|
||||
local body="$1"
|
||||
if [ -z "${TWILIO_ACCOUNT_SID:-}" ] || [ -z "${TWILIO_AUTH_TOKEN:-}" ] \
|
||||
|| [ -z "${TWILIO_PHONE_NUMBER:-}" ] || [ -z "${ALERT_SMS_TO:-}" ]; then
|
||||
log "ALERT (sms unconfigured): $body"
|
||||
return 1
|
||||
fi
|
||||
curl -s -m 15 -X POST \
|
||||
"https://api.twilio.com/2010-04-01/Accounts/$TWILIO_ACCOUNT_SID/Messages.json" \
|
||||
--data-urlencode "From=$TWILIO_PHONE_NUMBER" \
|
||||
--data-urlencode "To=$ALERT_SMS_TO" \
|
||||
--data-urlencode "Body=$body" \
|
||||
-u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" > /dev/null \
|
||||
&& log "SMS sent: $body" || log "SMS FAILED: $body"
|
||||
}
|
||||
|
||||
fails=$(cat "$FAILS_F" 2>/dev/null || echo 0)
|
||||
|
||||
if curl -sf -m 5 "$HEALTH_URL" > /dev/null 2>&1; then
|
||||
if [ -f "$ALERTED_F" ]; then
|
||||
send_sms "AVC phone agent RECOVERED ($(hostname), $(date '+%H:%M'))."
|
||||
rm -f "$ALERTED_F"
|
||||
fi
|
||||
[ "$fails" != "0" ] && log "healthy again after $fails failure(s)"
|
||||
echo 0 > "$FAILS_F"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
fails=$((fails + 1))
|
||||
echo "$fails" > "$FAILS_F"
|
||||
log "health check FAILED ($fails consecutive)"
|
||||
|
||||
if [ "$fails" -ge "$FAIL_THRESHOLD" ]; then
|
||||
now=$(date +%s)
|
||||
last=$(cat "$ALERTED_F" 2>/dev/null || echo 0)
|
||||
if [ $((now - last)) -ge $((ALERT_COOLDOWN_MIN * 60)) ]; then
|
||||
state=$(systemctl is-active avc-phone 2>/dev/null || echo unknown)
|
||||
send_sms "ALERT: AVC phone agent DOWN — $fails failed health checks ($(hostname), systemd: $state, $(date '+%H:%M')). Callers may be getting errors."
|
||||
echo "$now" > "$ALERTED_F"
|
||||
fi
|
||||
fi
|
||||
Reference in New Issue
Block a user