How to Build an AI Voicemail Transcription and Response System

An ai voicemail transcription and auto response system processes every missed call before you even check your phone. I build these for businesses where voicemails sit for hours because nobody has time to listen. The system transcribes the message, classifies the urgency, sends the caller a text confirmation, and routes the summary to the right person.

No more voicemail backlog. Every caller gets a response within minutes.

What You Need Before Starting

• A phone system with voicemail recording (Twilio, Vonage)
• Python 3.8+ with whisper, anthropic, and twilio
• SMS capability for auto-responses
• A routing system for different urgency levels

Step 1: Capture and Transcribe Voicemails

from flask import Flask, request, jsonify
import whisper
import requests
import os

app = Flask(__name__)
model = whisper.load_model("base")

@app.route("/voicemail", methods=["POST"])
def handle_voicemail():
    recording_url = request.form.get("RecordingUrl")
    caller = request.form.get("From")
    call_sid = request.form.get("CallSid")

    audio_path = download_recording(recording_url, call_sid)
    transcript = model.transcribe(audio_path)

    process_voicemail(caller, transcript["text"], call_sid)
    return "", 200

def download_recording(url, call_sid):
    response = requests.get(f"{url}.mp3", auth=(os.getenv("TWILIO_SID"), os.getenv("TWILIO_TOKEN")))
    path = f"voicemails/{call_sid}.mp3"
    os.makedirs("voicemails", exist_ok=True)
    with open(path, "wb") as f:
        f.write(response.content)
    return path

Step 2: Classify and Route

import anthropic
import json

client = anthropic.Anthropic()

def classify_voicemail(transcript):
    response = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=100,
        messages=[{
            "role": "user",
            "content": f"""Classify this voicemail. Respond with JSON only.

Voicemail: {transcript}

{{"urgency": "high/medium/low", "category": "sales/support/billing/personal/spam", "summary": "one sentence summary", "callback_needed": true/false}}"""
        }]
    )
    return json.loads(response.content[0].text)

ROUTING = {
    "sales": {"channel": "#sales-team", "sms_response": "Thanks for calling! Our sales team will get back to you within 2 hours."},
    "support": {"channel": "#support-queue", "sms_response": "We got your message. Our support team will call you back shortly."},
    "billing": {"channel": "#billing", "sms_response": "Thanks for reaching out about your account. We will review and respond within 24 hours."},
}

Step 3: Send Auto-Response

from twilio.rest import Client

twilio_client = Client(os.getenv("TWILIO_SID"), os.getenv("TWILIO_TOKEN"))

def send_auto_response(caller, category):
    route = ROUTING.get(category, ROUTING["support"])
    twilio_client.messages.create(
        body=route["sms_response"],
        from_=os.getenv("TWILIO_NUMBER"),
        to=caller
    )

Step 4: Process the Full Voicemail

def process_voicemail(caller, transcript, call_sid):
    classification = classify_voicemail(transcript)

    save_voicemail(call_sid, caller, transcript, classification)

    send_auto_response(caller, classification["category"])

    route = ROUTING.get(classification["category"], ROUTING["support"])
    notification = (
        f"New voicemail from {caller}\n"
        f"Urgency: {classification['urgency']}\n"
        f"Summary: {classification['summary']}\n"
        f"Callback needed: {'Yes' if classification['callback_needed'] else 'No'}\n"
        f"Full transcript: {transcript}"
    )
    send_slack_notification(route["channel"], notification)

    if classification["urgency"] == "high":
        send_sms_to_manager(notification)

Step 5: Track and Report

import sqlite3

def save_voicemail(call_sid, caller, transcript, classification):
    conn = sqlite3.connect("voicemails.db")
    conn.execute("""
        INSERT INTO voicemails (call_sid, caller, transcript, urgency, category, summary, callback_needed, received_at)
        VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
    """, (call_sid, caller, transcript, classification["urgency"], classification["category"],
          classification["summary"], classification["callback_needed"]))
    conn.commit()

def get_voicemail_report(days=7):
    conn = sqlite3.connect("voicemails.db")
    start = f"-{days} days"

    total = conn.execute("SELECT COUNT(*) FROM voicemails WHERE received_at > datetime('now', ?)", (start,)).fetchone()[0]
    by_category = conn.execute(
        "SELECT category, COUNT(*) FROM voicemails WHERE received_at > datetime('now', ?) GROUP BY category", (start,)
    ).fetchall()
    high_urgency = conn.execute(
        "SELECT COUNT(*) FROM voicemails WHERE urgency = 'high' AND received_at > datetime('now', ?)", (start,)
    ).fetchone()[0]
    callback_pending = conn.execute(
        "SELECT COUNT(*) FROM voicemails WHERE callback_needed = 1 AND status = 'pending' AND received_at > datetime('now', ?)", (start,)
    ).fetchone()[0]

    return {
        "total": total,
        "by_category": dict(by_category),
        "high_urgency": high_urgency,
        "callbacks_pending": callback_pending
    }

What to Build Next

Add callback tracking. When a team member calls back, log the callback and close the loop. Track average callback time by urgency level. High-urgency voicemails should get returned within 30 minutes. That metric tells you if the system is working.

Related Reading

• AI in Customer Service - voicemail automation as part of the support stack
• The Automation Decision Tree - voicemail processing as a high-value automation target
• The Real Cost of Manual Operations - the hidden cost of unlistened voicemails