How to Create an AI Voice IVR System

An ai voice ivr phone system automated with natural language understanding replaces the painful "press 1 for sales, press 2 for support" menu with a conversational experience. I build these for businesses handling 50+ inbound calls daily. The caller says what they need in plain language, the AI understands the intent, and routes or resolves it.

Callers get answers faster. Your team handles fewer repetitive calls.

What You Need Before Starting

• A telephony provider with API access (Twilio, Vonage)
• Python 3.8+ with twilio, whisper, and an AI model
• TTS capability for AI responses
• Call routing rules for your team

Step 1: Set Up Twilio Voice Webhook

from flask import Flask, request
from twilio.twiml.voice_response import VoiceResponse, Gather

app = Flask(__name__)

@app.route("/incoming-call", methods=["POST"])
def incoming_call():
    response = VoiceResponse()
    gather = Gather(
        input="speech",
        action="/process-speech",
        speech_timeout="auto",
        language="en-US"
    )
    gather.say("Hello, thanks for calling. How can I help you today?", voice="Polly.Joanna")
    response.append(gather)
    response.say("I did not catch that. Let me connect you to our team.")
    response.redirect("/connect-agent")
    return str(response)

Step 2: Process Speech with AI

import anthropic

client = anthropic.Anthropic()

INTENT_MAP = {
    "billing": {"action": "transfer", "number": "+15551234001", "department": "Billing"},
    "support": {"action": "transfer", "number": "+15551234002", "department": "Support"},
    "sales": {"action": "transfer", "number": "+15551234003", "department": "Sales"},
    "hours": {"action": "answer", "response": "We are open Monday through Friday, 9 AM to 6 PM Eastern time."},
    "address": {"action": "answer", "response": "We are located at 123 Main Street, Suite 200."},
}

@app.route("/process-speech", methods=["POST"])
def process_speech():
    speech_result = request.form.get("SpeechResult", "")
    caller = request.form.get("From", "unknown")

    intent = classify_intent(speech_result)
    return handle_intent(intent, speech_result, caller)

def classify_intent(speech_text):
    response = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=50,
        messages=[{
            "role": "user",
            "content": f"""Classify this caller's intent into ONE category: billing, support, sales, hours, address, other

Caller said: "{speech_text}"

Respond with ONLY the category name."""
        }]
    )
    return response.content[0].text.strip().lower()

Step 3: Handle Different Intents

from twilio.twiml.voice_response import VoiceResponse

def handle_intent(intent, speech_text, caller):
    response = VoiceResponse()
    action = INTENT_MAP.get(intent, {"action": "transfer", "number": "+15551234000"})

    if action["action"] == "answer":
        response.say(action["response"], voice="Polly.Joanna")
        gather = Gather(input="speech", action="/process-speech", speech_timeout="auto")
        gather.say("Is there anything else I can help with?", voice="Polly.Joanna")
        response.append(gather)

    elif action["action"] == "transfer":
        response.say(f"I will connect you to our {action.get('department', '')} team now.", voice="Polly.Joanna")
        response.dial(action["number"])

    log_call(caller, speech_text, intent, action["action"])
    return str(response)

Step 4: Add FAQ Handling

Answer common questions without transferring:

FAQ_RESPONSES = {
    "What are your hours?": "We are open Monday through Friday, 9 AM to 6 PM Eastern.",
    "Where are you located?": "Our office is at 123 Main Street, Suite 200.",
    "What is your return policy?": "You can return any item within 30 days for a full refund.",
}

def check_faq(speech_text):
    response = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=200,
        system=f"Answer the caller's question using ONLY these FAQs:\n{json.dumps(FAQ_RESPONSES)}\nIf the question is not in the FAQs, respond with 'TRANSFER'.",
        messages=[{"role": "user", "content": speech_text}]
    )
    answer = response.content[0].text
    if "TRANSFER" in answer:
        return None
    return answer

Step 5: Track Call Analytics

def log_call(caller, speech, intent, action):
    conn = sqlite3.connect("ivr.db")
    conn.execute("""
        INSERT INTO call_log (caller, speech_text, intent, action, called_at)
        VALUES (?, ?, ?, ?, datetime('now'))
    """, (caller, speech, intent, action))
    conn.commit()

def get_ivr_report(days=7):
    conn = sqlite3.connect("ivr.db")
    start = f"-{days} days"
    total = conn.execute("SELECT COUNT(*) FROM call_log WHERE called_at > datetime('now', ?)", (start,)).fetchone()[0]
    by_intent = conn.execute("SELECT intent, COUNT(*) FROM call_log WHERE called_at > datetime('now', ?) GROUP BY intent", (start,)).fetchall()
    answered = conn.execute("SELECT COUNT(*) FROM call_log WHERE action = 'answer' AND called_at > datetime('now', ?)", (start,)).fetchone()[0]

    return {"total_calls": total, "self_served": answered, "by_intent": dict(by_intent)}

What to Build Next

Add voicemail handling. When nobody answers, the AI takes a detailed message, transcribes it, classifies the urgency, and routes it to the right person with a summary. No more checking voicemail.

Related Reading

• AI in Customer Service - voice IVR as a customer service channel
• The Automation Decision Tree - when phone automation makes sense
• The Real Cost of Manual Operations - the cost of manually routing calls