Ops hardening: lead re-import, SMS health-watch, ambient office murmur
- scripts/reimport_leads.py drains appointment_requests.jsonl into Odoo (fallback leads were captured but invisible to staff forever); server.py now warns at startup when leads are pending. - scripts/healthwatch.sh (cron, every minute): after 3 consecutive /health failures sends a Twilio SMS to ALERT_SMS_TO (30-min cooldown) + a recovery SMS. systemd handles restarts; this makes a human aware when that isn't enough. Tested end-to-end (DOWN + RECOVERED SMS delivered). - Ambient office murmur (assets/office_ambience.wav, generated by scripts/make_ambience.py from low-passed unintelligible Kokoro babble + room tone) mixed continuously into outbound audio via pipecat SoundfileMixer, so callers hear a live office instead of dead silence while the agent thinks. AMBIENT_VOLUME/AMBIENT_ENABLED to tune/disable. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
tocmo0nlord
67
scripts/healthwatch.sh
Executable file
67
scripts/healthwatch.sh
Executable file
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# Health watch for the AVC phone agent. Run from cron every minute:
|
||||
# * * * * * /home/tocmo0nlord/avc-phone/scripts/healthwatch.sh
|
||||
#
|
||||
# Curls /health; after FAIL_THRESHOLD consecutive failures it sends ONE Twilio SMS to
|
||||
# ALERT_SMS_TO (re-alerts at most every ALERT_COOLDOWN_MIN), and sends a recovery SMS
|
||||
# when the service comes back. systemd already auto-restarts the process — this exists
|
||||
# so a human finds out when restarts aren't enough (crash loop, GPU wedged, box issues).
|
||||
# State lives in $STATE_DIR; activity is appended to healthwatch.log (gitignored).
|
||||
set -u
|
||||
|
||||
PROJ="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
set -a; . "$PROJ/.env" 2>/dev/null; set +a
|
||||
|
||||
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8200/health}"
|
||||
FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"
|
||||
ALERT_COOLDOWN_MIN="${ALERT_COOLDOWN_MIN:-30}"
|
||||
STATE_DIR="${STATE_DIR:-$HOME/.local/state/avc-healthwatch}"
|
||||
LOG="$PROJ/healthwatch.log"
|
||||
mkdir -p "$STATE_DIR"
|
||||
FAILS_F="$STATE_DIR/consecutive_fails"
|
||||
ALERTED_F="$STATE_DIR/alerted_at_epoch"
|
||||
|
||||
log() { echo "$(date '+%F %T') $*" >> "$LOG"; }
|
||||
|
||||
send_sms() {
|
||||
local body="$1"
|
||||
if [ -z "${TWILIO_ACCOUNT_SID:-}" ] || [ -z "${TWILIO_AUTH_TOKEN:-}" ] \
|
||||
|| [ -z "${TWILIO_PHONE_NUMBER:-}" ] || [ -z "${ALERT_SMS_TO:-}" ]; then
|
||||
log "ALERT (sms unconfigured): $body"
|
||||
return 1
|
||||
fi
|
||||
curl -s -m 15 -X POST \
|
||||
"https://api.twilio.com/2010-04-01/Accounts/$TWILIO_ACCOUNT_SID/Messages.json" \
|
||||
--data-urlencode "From=$TWILIO_PHONE_NUMBER" \
|
||||
--data-urlencode "To=$ALERT_SMS_TO" \
|
||||
--data-urlencode "Body=$body" \
|
||||
-u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" > /dev/null \
|
||||
&& log "SMS sent: $body" || log "SMS FAILED: $body"
|
||||
}
|
||||
|
||||
fails=$(cat "$FAILS_F" 2>/dev/null || echo 0)
|
||||
|
||||
if curl -sf -m 5 "$HEALTH_URL" > /dev/null 2>&1; then
|
||||
if [ -f "$ALERTED_F" ]; then
|
||||
send_sms "AVC phone agent RECOVERED ($(hostname), $(date '+%H:%M'))."
|
||||
rm -f "$ALERTED_F"
|
||||
fi
|
||||
[ "$fails" != "0" ] && log "healthy again after $fails failure(s)"
|
||||
echo 0 > "$FAILS_F"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
fails=$((fails + 1))
|
||||
echo "$fails" > "$FAILS_F"
|
||||
log "health check FAILED ($fails consecutive)"
|
||||
|
||||
if [ "$fails" -ge "$FAIL_THRESHOLD" ]; then
|
||||
now=$(date +%s)
|
||||
last=$(cat "$ALERTED_F" 2>/dev/null || echo 0)
|
||||
if [ $((now - last)) -ge $((ALERT_COOLDOWN_MIN * 60)) ]; then
|
||||
state=$(systemctl is-active avc-phone 2>/dev/null || echo unknown)
|
||||
send_sms "ALERT: AVC phone agent DOWN — $fails failed health checks ($(hostname), systemd: $state, $(date '+%H:%M')). Callers may be getting errors."
|
||||
echo "$now" > "$ALERTED_F"
|
||||
fi
|
||||
fi
|
||||
86
scripts/make_ambience.py
Normal file
86
scripts/make_ambience.py
Normal file
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate assets/office_ambience.wav — low, unintelligible office murmur.
|
||||
|
||||
Mixed continuously into the outbound call audio (SoundfileMixer) so the caller hears a
|
||||
live-office room tone instead of dead digital silence while the agent thinks. Built from
|
||||
Kokoro voices speaking mundane phrases, overlapped and heavily low-pass filtered so no
|
||||
words are intelligible (nothing that could be mistaken for real conversation/PHI), plus
|
||||
a touch of room noise. Output: 60s mono 16 kHz PCM16 loop, quiet by design (the mixer
|
||||
volume scales it further).
|
||||
|
||||
Run inside the pipecat venv: python scripts/make_ambience.py
|
||||
"""
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from kokoro_onnx import Kokoro
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PROJ = os.path.dirname(HERE)
|
||||
MODEL_DIR = os.environ.get("KOKORO_MODEL_DIR", "/home/tocmo0nlord/pipecat-run/models")
|
||||
OUT = os.path.join(PROJ, "assets", "office_ambience.wav")
|
||||
|
||||
SR = 16000 # match PIPELINE_SAMPLE_RATE (transport output rate)
|
||||
DUR = 60.0 # loop length in seconds
|
||||
RNG = np.random.default_rng(42)
|
||||
|
||||
PHRASES = [
|
||||
"Okay, so I'll move that over to Thursday and send the reminder out this afternoon.",
|
||||
"Could you pull the file for the three o'clock? I think it's already up front.",
|
||||
"Yes, we got the shipment in this morning, it's in the back on the second shelf.",
|
||||
"Let me transfer you over, one moment please, thank you so much for holding.",
|
||||
"The afternoon looks pretty full but the morning still has a couple of openings.",
|
||||
"I'll leave a note for the doctor and we'll follow up first thing tomorrow.",
|
||||
"They said the delivery should arrive before noon so we should be all set.",
|
||||
"Can you double check the calendar for next Tuesday? I think there's a conflict.",
|
||||
]
|
||||
VOICES = ["af_bella", "am_michael", "bf_emma", "am_adam", "af_nicole", "bm_george"]
|
||||
|
||||
|
||||
def lowpass(x, cutoff_hz, sr):
|
||||
"""Simple FFT brick-wall low-pass — fine for ambience shaping."""
|
||||
X = np.fft.rfft(x)
|
||||
freqs = np.fft.rfftfreq(len(x), 1 / sr)
|
||||
X[freqs > cutoff_hz] = 0
|
||||
return np.fft.irfft(X, n=len(x))
|
||||
|
||||
|
||||
def main():
|
||||
k = Kokoro(os.path.join(MODEL_DIR, "kokoro-v1.0.onnx"),
|
||||
os.path.join(MODEL_DIR, "voices-v1.0.bin"))
|
||||
bed = np.zeros(int(SR * DUR), dtype=np.float64)
|
||||
|
||||
# ~18 murmured utterances scattered through the minute, different voices/speeds.
|
||||
for i in range(18):
|
||||
text = PHRASES[i % len(PHRASES)]
|
||||
voice = VOICES[i % len(VOICES)]
|
||||
samples, sr0 = k.create(text, voice=voice, speed=float(RNG.uniform(0.9, 1.1)))
|
||||
# Resample to SR by linear interpolation (quality is irrelevant post-filter).
|
||||
t_src = np.arange(len(samples)) / sr0
|
||||
t_dst = np.arange(int(len(samples) * SR / sr0)) / SR
|
||||
s = np.interp(t_dst, t_src, samples.astype(np.float64))
|
||||
s = lowpass(s, 900, SR) # muffle: through-the-wall murmur
|
||||
s *= RNG.uniform(0.25, 0.5) # each talker is quiet and uneven
|
||||
start = int(RNG.uniform(0, DUR - len(s) / SR) * SR)
|
||||
bed[start:start + len(s)] += s
|
||||
|
||||
# Gentle room tone under the voices (shaped noise), so pauses aren't pure silence.
|
||||
noise = lowpass(RNG.standard_normal(len(bed)), 400, SR) * 0.02
|
||||
bed += noise
|
||||
|
||||
# Loop seamlessly: crossfade the last second into the first.
|
||||
xf = SR
|
||||
ramp = np.linspace(0, 1, xf)
|
||||
bed[:xf] = bed[:xf] * ramp + bed[-xf:] * (1 - ramp)
|
||||
bed = bed[:-xf]
|
||||
|
||||
# Normalize conservatively — this is a QUIET bed; mixer volume scales it down further.
|
||||
bed = bed / (np.abs(bed).max() + 1e-9) * 0.35
|
||||
os.makedirs(os.path.dirname(OUT), exist_ok=True)
|
||||
sf.write(OUT, bed.astype(np.float32), SR, subtype="PCM_16")
|
||||
print(f"wrote {OUT}: {len(bed)/SR:.1f}s @ {SR}Hz, peak {np.abs(bed).max():.2f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
96
scripts/reimport_leads.py
Executable file
96
scripts/reimport_leads.py
Executable file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Re-import fallback leads (appointment_requests.jsonl) into Odoo.
|
||||
|
||||
When Odoo is unreachable during a call, the lead is saved to the JSONL fallback so it's
|
||||
never lost — but nothing pushed it into Odoo afterwards, so staff would never see it.
|
||||
This script drains the file: each record is written to Odoo; successes are removed,
|
||||
failures are kept for the next run. server.py also warns at startup when the file has
|
||||
pending leads.
|
||||
|
||||
Usage (loads ../.env itself):
|
||||
python scripts/reimport_leads.py # import pending leads
|
||||
python scripts/reimport_leads.py --dry-run # show what would be imported
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
PROJ = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJ))
|
||||
|
||||
|
||||
def load_env(path):
|
||||
"""Minimal .env loader (KEY=VALUE lines; doesn't override existing env)."""
|
||||
if not path.exists():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, v = line.split("=", 1)
|
||||
v = v.split(" #")[0].strip().strip("'\"")
|
||||
os.environ.setdefault(k.strip(), v)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
load_env(PROJ / ".env")
|
||||
jsonl = Path(os.environ.get("REQUESTS_LOG", PROJ / "appointment_requests.jsonl"))
|
||||
if not jsonl.exists() or jsonl.stat().st_size == 0:
|
||||
print("No pending fallback leads — nothing to do.")
|
||||
return 0
|
||||
|
||||
from odoo_client import create_appointment_request # after env is loaded
|
||||
|
||||
kept, imported = [], 0
|
||||
lines = [ln for ln in jsonl.read_text().splitlines() if ln.strip()]
|
||||
print(f"{len(lines)} pending lead(s) in {jsonl}")
|
||||
for ln in lines:
|
||||
try:
|
||||
rec = json.loads(ln)
|
||||
except json.JSONDecodeError:
|
||||
print(f" SKIP (unparseable, kept): {ln[:80]}")
|
||||
kept.append(ln)
|
||||
continue
|
||||
kind = rec.get("kind", "appointment")
|
||||
label = f"{kind}: {rec.get('patient_name')} / {rec.get('reason')} ({rec.get('ts')})"
|
||||
if args.dry_run:
|
||||
print(f" would import {label}")
|
||||
kept.append(ln)
|
||||
continue
|
||||
try:
|
||||
if kind == "callback":
|
||||
reason = rec.get("reason") or "callback request"
|
||||
else:
|
||||
reason = f"[{rec.get('location') or 'location TBD'}] {rec.get('reason') or ''}".strip()
|
||||
model, rec_id = create_appointment_request(
|
||||
patient_name=rec.get("patient_name"),
|
||||
callback_number=rec.get("callback_number"),
|
||||
reason=f"{reason} (re-imported from fallback)",
|
||||
preferred_time=rec.get("preferred_time"),
|
||||
insurance=rec.get("insurance"),
|
||||
call_sid=rec.get("call_sid"),
|
||||
kind=kind,
|
||||
)
|
||||
imported += 1
|
||||
print(f" OK -> Odoo {model} id={rec_id}: {label}")
|
||||
except Exception as e:
|
||||
print(f" FAILED (kept for next run): {label} — {e}")
|
||||
kept.append(ln)
|
||||
|
||||
if not args.dry_run:
|
||||
if kept:
|
||||
jsonl.write_text("\n".join(kept) + "\n")
|
||||
else:
|
||||
jsonl.unlink()
|
||||
print(f"Done: {imported} imported, {len(kept)} kept.")
|
||||
return 0 if not kept or args.dry_run else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user