Ops hardening: lead re-import, SMS health-watch, ambient office murmur

- scripts/reimport_leads.py drains appointment_requests.jsonl into Odoo
  (fallback leads were captured but invisible to staff forever); server.py
  now warns at startup when leads are pending.
- scripts/healthwatch.sh (cron, every minute): after 3 consecutive /health
  failures sends a Twilio SMS to ALERT_SMS_TO (30-min cooldown) + a recovery
  SMS. systemd handles restarts; this makes a human aware when that isn't
  enough. Tested end-to-end (DOWN + RECOVERED SMS delivered).
- Ambient office murmur (assets/office_ambience.wav, generated by
  scripts/make_ambience.py from low-passed unintelligible Kokoro babble +
  room tone) mixed continuously into outbound audio via pipecat
  SoundfileMixer, so callers hear a live office instead of dead silence
  while the agent thinks. AMBIENT_VOLUME/AMBIENT_ENABLED to tune/disable.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

.env.example

@@ -75,3 +75,13 @@ VAD_STOP_SECS=0.5
 # checklist into the system prompt each turn + merges VAD-fragmented user turns, so the
 # local 8B stops re-asking for name/reason/phone. Default: on for ollama, off for anthropic.
 #CALL_STATE_TRACKING=true
+# Ambient office murmur mixed into outbound audio (masks reply latency; file from
+# scripts/make_ambience.py). AMBIENT_VOLUME=0 or AMBIENT_ENABLED=false disables.
+#AMBIENT_SOUND=assets/office_ambience.wav
+#AMBIENT_VOLUME=0.10
+# Hard timeout for every Odoo XML-RPC round-trip (a HUNG Odoo must not hold call slots).
+#ODOO_TIMEOUT_SECS=15
+# Health-watch alerting (scripts/healthwatch.sh via cron): SMS on sustained /health
+# failure. From = your Twilio number; To = the phone that should get the alert.
+TWILIO_PHONE_NUMBER=+1...
+ALERT_SMS_TO=+1...

.gitignore

@@ -21,3 +21,4 @@ venv/
 # OS / editor
 .DS_Store
 *.swp
+healthwatch.log

bot.py

@@ -43,6 +43,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
     LLMContextAggregatorPair,
     LLMUserAggregatorParams,
 )
+from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer
 from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
 from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
 from pipecat.turns.user_start import (
@@ -149,6 +150,19 @@ CALL_STATE_TRACKING = (
     if _call_state_env is not None
     else (LLM_PROVIDER == "ollama")
 )
+# Ambient office murmur mixed continuously into the OUTBOUND call audio, so the caller
+# hears a live-office room tone instead of dead digital silence while the agent thinks
+# (masks per-turn latency). File generated by scripts/make_ambience.py — unintelligible
+# by construction (low-passed). Mixed by pipecat's SoundfileMixer at the output transport,
+# which keeps streaming it even when the bot isn't speaking. Volume is the mixer's scale
+# on top of an already-quiet bed; 0.0/missing file/AMBIENT_ENABLED=false disables.
+AMBIENT_SOUND = os.environ.get("AMBIENT_SOUND", os.path.join(HERE, "assets", "office_ambience.wav"))
+AMBIENT_VOLUME = float(os.environ.get("AMBIENT_VOLUME", "0.10"))
+AMBIENT_ENABLED = (
+    os.environ.get("AMBIENT_ENABLED", "true").lower() not in ("false", "0", "no")
+    and AMBIENT_VOLUME > 0
+    and os.path.isfile(AMBIENT_SOUND)
+)
 # Record each call to a stereo WAV (caller = left, agent = right) for review/debugging.
 RECORD_CALLS = os.environ.get("RECORD_CALLS", "true").lower() not in ("false", "0", "no")
 RECORDINGS_DIR = os.environ.get("RECORDINGS_DIR", os.path.join(HERE, "recordings"))
@@ -836,6 +850,11 @@ async def run_agent(transport, caller_number=None, call_sid=None, do_capture=Tru
 
 async def run_call(websocket, serializer: TwilioFrameSerializer, caller_number=None, call_sid=None):
     """Phone entrypoint: wrap the Twilio Media Stream in a transport, run the shared agent."""
+    mixer = SoundfileMixer(
+        sound_files={"office": AMBIENT_SOUND},
+        default_sound="office",
+        volume=AMBIENT_VOLUME,
+    ) if AMBIENT_ENABLED else None
     transport = FastAPIWebsocketTransport(
         websocket=websocket,
         params=FastAPIWebsocketParams(
@@ -845,6 +864,7 @@ async def run_call(websocket, serializer: TwilioFrameSerializer, caller_number=N
             audio_out_sample_rate=PIPELINE_SAMPLE_RATE,
             add_wav_header=False,
             serializer=serializer,
+            audio_out_mixer=mixer,  # constant low office murmur masks reply latency
         ),
     )
     await run_agent(transport, caller_number=caller_number, call_sid=call_sid, do_capture=True)

scripts/healthwatch.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Health watch for the AVC phone agent. Run from cron every minute:
+#   * * * * * /home/tocmo0nlord/avc-phone/scripts/healthwatch.sh
+#
+# Curls /health; after FAIL_THRESHOLD consecutive failures it sends ONE Twilio SMS to
+# ALERT_SMS_TO (re-alerts at most every ALERT_COOLDOWN_MIN), and sends a recovery SMS
+# when the service comes back. systemd already auto-restarts the process — this exists
+# so a human finds out when restarts aren't enough (crash loop, GPU wedged, box issues).
+# State lives in $STATE_DIR; activity is appended to healthwatch.log (gitignored).
+set -u
+
+PROJ="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+# shellcheck disable=SC1091
+set -a; . "$PROJ/.env" 2>/dev/null; set +a
+
+HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8200/health}"
+FAIL_THRESHOLD="${FAIL_THRESHOLD:-3}"
+ALERT_COOLDOWN_MIN="${ALERT_COOLDOWN_MIN:-30}"
+STATE_DIR="${STATE_DIR:-$HOME/.local/state/avc-healthwatch}"
+LOG="$PROJ/healthwatch.log"
+mkdir -p "$STATE_DIR"
+FAILS_F="$STATE_DIR/consecutive_fails"
+ALERTED_F="$STATE_DIR/alerted_at_epoch"
+
+log() { echo "$(date '+%F %T') $*" >> "$LOG"; }
+
+send_sms() {
+    local body="$1"
+    if [ -z "${TWILIO_ACCOUNT_SID:-}" ] || [ -z "${TWILIO_AUTH_TOKEN:-}" ] \
+       || [ -z "${TWILIO_PHONE_NUMBER:-}" ] || [ -z "${ALERT_SMS_TO:-}" ]; then
+        log "ALERT (sms unconfigured): $body"
+        return 1
+    fi
+    curl -s -m 15 -X POST \
+        "https://api.twilio.com/2010-04-01/Accounts/$TWILIO_ACCOUNT_SID/Messages.json" \
+        --data-urlencode "From=$TWILIO_PHONE_NUMBER" \
+        --data-urlencode "To=$ALERT_SMS_TO" \
+        --data-urlencode "Body=$body" \
+        -u "$TWILIO_ACCOUNT_SID:$TWILIO_AUTH_TOKEN" > /dev/null \
+        && log "SMS sent: $body" || log "SMS FAILED: $body"
+}
+
+fails=$(cat "$FAILS_F" 2>/dev/null || echo 0)
+
+if curl -sf -m 5 "$HEALTH_URL" > /dev/null 2>&1; then
+    if [ -f "$ALERTED_F" ]; then
+        send_sms "AVC phone agent RECOVERED ($(hostname), $(date '+%H:%M'))."
+        rm -f "$ALERTED_F"
+    fi
+    [ "$fails" != "0" ] && log "healthy again after $fails failure(s)"
+    echo 0 > "$FAILS_F"
+    exit 0
+fi
+
+fails=$((fails + 1))
+echo "$fails" > "$FAILS_F"
+log "health check FAILED ($fails consecutive)"
+
+if [ "$fails" -ge "$FAIL_THRESHOLD" ]; then
+    now=$(date +%s)
+    last=$(cat "$ALERTED_F" 2>/dev/null || echo 0)
+    if [ $((now - last)) -ge $((ALERT_COOLDOWN_MIN * 60)) ]; then
+        state=$(systemctl is-active avc-phone 2>/dev/null || echo unknown)
+        send_sms "ALERT: AVC phone agent DOWN — $fails failed health checks ($(hostname), systemd: $state, $(date '+%H:%M')). Callers may be getting errors."
+        echo "$now" > "$ALERTED_F"
+    fi
+fi

scripts/make_ambience.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Generate assets/office_ambience.wav — low, unintelligible office murmur.
+
+Mixed continuously into the outbound call audio (SoundfileMixer) so the caller hears a
+live-office room tone instead of dead digital silence while the agent thinks. Built from
+Kokoro voices speaking mundane phrases, overlapped and heavily low-pass filtered so no
+words are intelligible (nothing that could be mistaken for real conversation/PHI), plus
+a touch of room noise. Output: 60s mono 16 kHz PCM16 loop, quiet by design (the mixer
+volume scales it further).
+
+Run inside the pipecat venv: python scripts/make_ambience.py
+"""
+import os
+
+import numpy as np
+import soundfile as sf
+from kokoro_onnx import Kokoro
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PROJ = os.path.dirname(HERE)
+MODEL_DIR = os.environ.get("KOKORO_MODEL_DIR", "/home/tocmo0nlord/pipecat-run/models")
+OUT = os.path.join(PROJ, "assets", "office_ambience.wav")
+
+SR = 16000          # match PIPELINE_SAMPLE_RATE (transport output rate)
+DUR = 60.0          # loop length in seconds
+RNG = np.random.default_rng(42)
+
+PHRASES = [
+    "Okay, so I'll move that over to Thursday and send the reminder out this afternoon.",
+    "Could you pull the file for the three o'clock? I think it's already up front.",
+    "Yes, we got the shipment in this morning, it's in the back on the second shelf.",
+    "Let me transfer you over, one moment please, thank you so much for holding.",
+    "The afternoon looks pretty full but the morning still has a couple of openings.",
+    "I'll leave a note for the doctor and we'll follow up first thing tomorrow.",
+    "They said the delivery should arrive before noon so we should be all set.",
+    "Can you double check the calendar for next Tuesday? I think there's a conflict.",
+]
+VOICES = ["af_bella", "am_michael", "bf_emma", "am_adam", "af_nicole", "bm_george"]
+
+
+def lowpass(x, cutoff_hz, sr):
+    """Simple FFT brick-wall low-pass — fine for ambience shaping."""
+    X = np.fft.rfft(x)
+    freqs = np.fft.rfftfreq(len(x), 1 / sr)
+    X[freqs > cutoff_hz] = 0
+    return np.fft.irfft(X, n=len(x))
+
+
+def main():
+    k = Kokoro(os.path.join(MODEL_DIR, "kokoro-v1.0.onnx"),
+               os.path.join(MODEL_DIR, "voices-v1.0.bin"))
+    bed = np.zeros(int(SR * DUR), dtype=np.float64)
+
+    # ~18 murmured utterances scattered through the minute, different voices/speeds.
+    for i in range(18):
+        text = PHRASES[i % len(PHRASES)]
+        voice = VOICES[i % len(VOICES)]
+        samples, sr0 = k.create(text, voice=voice, speed=float(RNG.uniform(0.9, 1.1)))
+        # Resample to SR by linear interpolation (quality is irrelevant post-filter).
+        t_src = np.arange(len(samples)) / sr0
+        t_dst = np.arange(int(len(samples) * SR / sr0)) / SR
+        s = np.interp(t_dst, t_src, samples.astype(np.float64))
+        s = lowpass(s, 900, SR)                # muffle: through-the-wall murmur
+        s *= RNG.uniform(0.25, 0.5)            # each talker is quiet and uneven
+        start = int(RNG.uniform(0, DUR - len(s) / SR) * SR)
+        bed[start:start + len(s)] += s
+
+    # Gentle room tone under the voices (shaped noise), so pauses aren't pure silence.
+    noise = lowpass(RNG.standard_normal(len(bed)), 400, SR) * 0.02
+    bed += noise
+
+    # Loop seamlessly: crossfade the last second into the first.
+    xf = SR
+    ramp = np.linspace(0, 1, xf)
+    bed[:xf] = bed[:xf] * ramp + bed[-xf:] * (1 - ramp)
+    bed = bed[:-xf]
+
+    # Normalize conservatively — this is a QUIET bed; mixer volume scales it down further.
+    bed = bed / (np.abs(bed).max() + 1e-9) * 0.35
+    os.makedirs(os.path.dirname(OUT), exist_ok=True)
+    sf.write(OUT, bed.astype(np.float32), SR, subtype="PCM_16")
+    print(f"wrote {OUT}: {len(bed)/SR:.1f}s @ {SR}Hz, peak {np.abs(bed).max():.2f}")
+
+
+if __name__ == "__main__":
+    main()

scripts/reimport_leads.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Re-import fallback leads (appointment_requests.jsonl) into Odoo.
+
+When Odoo is unreachable during a call, the lead is saved to the JSONL fallback so it's
+never lost — but nothing pushed it into Odoo afterwards, so staff would never see it.
+This script drains the file: each record is written to Odoo; successes are removed,
+failures are kept for the next run. server.py also warns at startup when the file has
+pending leads.
+
+Usage (loads ../.env itself):
+    python scripts/reimport_leads.py            # import pending leads
+    python scripts/reimport_leads.py --dry-run  # show what would be imported
+"""
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+PROJ = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(PROJ))
+
+
+def load_env(path):
+    """Minimal .env loader (KEY=VALUE lines; doesn't override existing env)."""
+    if not path.exists():
+        return
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        v = v.split(" #")[0].strip().strip("'\"")
+        os.environ.setdefault(k.strip(), v)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    load_env(PROJ / ".env")
+    jsonl = Path(os.environ.get("REQUESTS_LOG", PROJ / "appointment_requests.jsonl"))
+    if not jsonl.exists() or jsonl.stat().st_size == 0:
+        print("No pending fallback leads — nothing to do.")
+        return 0
+
+    from odoo_client import create_appointment_request  # after env is loaded
+
+    kept, imported = [], 0
+    lines = [ln for ln in jsonl.read_text().splitlines() if ln.strip()]
+    print(f"{len(lines)} pending lead(s) in {jsonl}")
+    for ln in lines:
+        try:
+            rec = json.loads(ln)
+        except json.JSONDecodeError:
+            print(f"  SKIP (unparseable, kept): {ln[:80]}")
+            kept.append(ln)
+            continue
+        kind = rec.get("kind", "appointment")
+        label = f"{kind}: {rec.get('patient_name')} / {rec.get('reason')} ({rec.get('ts')})"
+        if args.dry_run:
+            print(f"  would import {label}")
+            kept.append(ln)
+            continue
+        try:
+            if kind == "callback":
+                reason = rec.get("reason") or "callback request"
+            else:
+                reason = f"[{rec.get('location') or 'location TBD'}] {rec.get('reason') or ''}".strip()
+            model, rec_id = create_appointment_request(
+                patient_name=rec.get("patient_name"),
+                callback_number=rec.get("callback_number"),
+                reason=f"{reason} (re-imported from fallback)",
+                preferred_time=rec.get("preferred_time"),
+                insurance=rec.get("insurance"),
+                call_sid=rec.get("call_sid"),
+                kind=kind,
+            )
+            imported += 1
+            print(f"  OK -> Odoo {model} id={rec_id}: {label}")
+        except Exception as e:
+            print(f"  FAILED (kept for next run): {label} — {e}")
+            kept.append(ln)
+
+    if not args.dry_run:
+        if kept:
+            jsonl.write_text("\n".join(kept) + "\n")
+        else:
+            jsonl.unlink()
+        print(f"Done: {imported} imported, {len(kept)} kept.")
+    return 0 if not kept or args.dry_run else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

server.py

@@ -80,6 +80,18 @@ async def lifespan(app: FastAPI):
         logger.info(f"Warmed + pinned Ollama model {model} (keep_alive=-1)")
     except Exception as e:
         logger.warning(f"LLM warmup failed (first call may be slow): {e!r}")
+    # Pending fallback leads = requests captured while Odoo was down. They are invisible
+    # to staff until drained — shout about it on every startup.
+    try:
+        from practice import REQUESTS_LOG
+        if os.path.isfile(REQUESTS_LOG) and os.path.getsize(REQUESTS_LOG) > 0:
+            n = sum(1 for ln in open(REQUESTS_LOG) if ln.strip())
+            logger.warning(
+                f"{n} lead(s) pending in {REQUESTS_LOG} — staff can't see these! "
+                f"Run: python scripts/reimport_leads.py"
+            )
+    except Exception:
+        logger.exception("fallback-lead check failed")
     yield