aishastr.live

Agentic Architecture Pattern: Audio Text

Written by

admin

in


    """
Pattern 222: Audio-Text Pattern

Multi-modal agent that processes both audio and textual information,
enabling speech understanding, emotion detection, and contextual responses.
"""

from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI


class AudioFeatureType(Enum):
    """Types of audio features."""
    SPEECH = "speech"
    EMOTION = "emotion"
    AMBIENT = "ambient"
    MUSIC = "music"


@dataclass
class AudioInput:
    """Represents audio input (simulated)."""
    transcript: str  # In real impl, would come from ASR
    duration: float  # seconds
    features: Dict[str, Any] = field(default_factory=dict)
    timestamp: datetime = field(default_factory=datetime.now)


@dataclass
class AudioFeatures:
    """Extracted audio features."""
    speech_content: str
    emotion: str  # happy, sad, angry, neutral, etc.
    tone: str  # formal, casual, urgent, etc.
    pace: str  # fast, normal, slow
    confidence: float = 0.0


class AudioAnalyzer:
    """Analyzes audio characteristics."""
    
    def __init__(self, llm: ChatOpenAI):
        self.llm = llm
    
    def analyze(self, audio: AudioInput) -> AudioFeatures:
        """Extract audio features from transcript."""
        prompt = ChatPromptTemplate.from_messages([
            ("system", """You are an audio analysis expert. Analyze the speech for:
            1. Emotional tone (happy, sad, angry, neutral, excited, etc.)
            2. Speaking tone (formal, casual, urgent, calm, etc.)
            3. Pace (fast, normal, slow)"""),
            ("human", "Speech transcript: '{transcript}'\nProvide analysis.")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({"transcript": audio.transcript})
        content = str(response.content).lower()
        
        # Parse response
        emotion = "neutral"
        if "happy" in content or "joy" in content:
            emotion = "happy"
        elif "sad" in content or "melancholy" in content:
            emotion = "sad"
        elif "angry" in content or "frustrated" in content:
            emotion = "angry"
        elif "excited" in content:
            emotion = "excited"
        
        tone = "casual" if "casual" in content else "formal" if "formal" in content else "neutral"
        pace = "fast" if "fast" in content else "slow" if "slow" in content else "normal"
        
        return AudioFeatures(
            speech_content=audio.transcript,
            emotion=emotion,
            tone=tone,
            pace=pace,
            confidence=0.85
        )


class AudioTextFusion:
    """Fuses audio features with text understanding."""
    
    def __init__(self, llm: ChatOpenAI):
        self.llm = llm
    
    def fuse(self, audio_features: AudioFeatures, text_query: Optional[str] = None) -> Dict[str, Any]:
        """Create unified audio-text understanding."""
        prompt = ChatPromptTemplate.from_messages([
            ("system", "Combine audio characteristics with text content for contextual understanding."),
            ("human", """Speech: "{speech}"
Emotion: {emotion}
Tone: {tone}
Pace: {pace}
{query_part}

Provide contextual understanding considering both content and delivery.""")
        ])
        
        query_part = f"\nAdditional query: {text_query}" if text_query else ""
        
        chain = prompt | self.llm
        response = chain.invoke({
            "speech": audio_features.speech_content,
            "emotion": audio_features.emotion,
            "tone": audio_features.tone,
            "pace": audio_features.pace,
            "query_part": query_part
        })
        
        return {
            "understanding": str(response.content),
            "emotion": audio_features.emotion,
            "tone": audio_features.tone
        }


class AudioTextPattern:
    """
    Audio-Text Pattern - Processes combined audio and textual inputs.
    
    Enables speech understanding with emotion and prosody awareness,
    suitable for voice assistants and conversational AI.
    """
    
    def __init__(self, model_name: str = "gpt-4"):
        self.llm = ChatOpenAI(model=model_name, temperature=0.3)
        self.audio_analyzer = AudioAnalyzer(self.llm)
        self.fusion = AudioTextFusion(self.llm)
        self.conversation_history: List[Dict[str, Any]] = []
    
    def process(self, audio: AudioInput, text_query: Optional[str] = None) -> Dict[str, Any]:
        """Process audio with optional text query."""
        
        # Analyze audio
        audio_features = self.audio_analyzer.analyze(audio)
        
        # Fuse with text
        fused = self.fusion.fuse(audio_features, text_query)
        
        # Generate response considering audio context
        prompt = ChatPromptTemplate.from_messages([
            ("system", """You are an empathetic audio-text assistant. Respond appropriately 
            considering both the content and the emotional/tonal delivery."""),
            ("human", """Understanding: {understanding}
Detected emotion: {emotion}
Speaking tone: {tone}

Generate an appropriate response.""")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({
            "understanding": fused["understanding"],
            "emotion": fused["emotion"],
            "tone": fused["tone"]
        })
        
        result = {
            "response": str(response.content),
            "detected_emotion": audio_features.emotion,
            "detected_tone": audio_features.tone,
            "speech_pace": audio_features.pace,
            "confidence": audio_features.confidence
        }
        
        self.conversation_history.append(result)
        return result
    
    def run(self, speech_transcript: str, duration: float = 3.0, text_query: Optional[str] = None) -> Dict[str, Any]:
        """Main execution method."""
        audio = AudioInput(
            transcript=speech_transcript,
            duration=duration
        )
        
        result = self.process(audio, text_query)
        
        return {
            "pattern": "Audio-Text Pattern",
            **result
        }


# Example usage
if __name__ == "__main__":
    print("=== Audio-Text Pattern Demo ===\n")
    
    pattern = AudioTextPattern()
    
    # Example 1: Excited speech
    result1 = pattern.run(
        speech_transcript="I just got the job! This is amazing! I can't believe it!",
        duration=3.5
    )
    
    print("Example 1: Excited announcement")
    print(f"Detected emotion: {result1['detected_emotion']}")
    print(f"Detected tone: {result1['detected_tone']}")
    print(f"Response: {result1['response'][:150]}...\n")
    
    # Example 2: Urgent request
    result2 = pattern.run(
        speech_transcript="I need help right away. The server is down and customers can't access the site.",
        duration=4.0,
        text_query="What should I do first?"
    )
    
    print("Example 2: Urgent technical issue")
    print(f"Detected emotion: {result2['detected_emotion']}")
    print(f"Detected tone: {result2['detected_tone']}")
    print(f"Response: {result2['response'][:150]}...\n")
    
    # Example 3: Calm inquiry
    result3 = pattern.run(
        speech_transcript="Could you please explain how the new system works? I'd like to understand it better.",
        duration=3.0
    )
    
    print("Example 3: Calm inquiry")
    print(f"Detected emotion: {result3['detected_emotion']}")
    print(f"Detected tone: {result3['detected_tone']}")
    print(f"Response: {result3['response'][:150]}...\n")
    
    print(f"Total conversations: {len(pattern.conversation_history)}")

Agentic Architecture Pattern: Audio Text

Pattern 222: 222 Audio Text – Line-by-Line Explanation

Overview

Pattern 222: Audio-Text Pattern

Multi-modal agent that processes both audio and textual information, enabling speech understanding, emotion detection, and contextual responses.

Code Structure Analysis

File Information

  • Pattern Number: 222
  • Pattern Name: 222 Audio Text
  • File Path: patterns\multi_modal\222_audio_text.py
  • Total Lines: 231

Detailed Code Explanation

Module Docstring

"""
Pattern 222: Audio-Text Pattern

Multi-modal agent that processes both audio and textual information,
enabling speech understanding, emotion detection, and contextual responses.
"""

Explanation: This module docstring describes the purpose and key features of the 222 Audio Text pattern.

Imports Section

from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI  # First 20 imports shown

Explanation: The module imports necessary dependencies: – from typing import Dict, Any, List, Optional: Type hints for better code documentation – from dataclasses import dataclass, field: Data class decorators for clean data structures – from datetime import datetime: Date and time utilities – from enum import Enum: Enumeration support for fixed sets of values – from langchain_core.prompts import ChatPromptTemplate: LangChain framework integration – from langchain_openai import ChatOpenAI: LangChain framework integration

Enumerations

AudioFeatureType (Line 16)

class AudioFeatureType(Enum):
    SPEECH
    EMOTION
    AMBIENT
    MUSIC

Purpose: Types of audio features.

Values: – SPEECH: Represents speech – EMOTION: Represents emotion – AMBIENT: Represents ambient – MUSIC: Represents music

Classes

AudioFeatureType (Line 16)

Description: Types of audio features.

AudioInput (Line 25)

Description: Represents audio input (simulated).

AudioFeatures (Line 34)

Description: Extracted audio features.

AudioAnalyzer (Line 43)

Description: Analyzes audio characteristics.

Methods:

__init__ (Line 46)
def __init__(self, llm):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – llm: Parameter for llm

analyze (Line 49)
def analyze(self, audio):
    """Extract audio features from transcript."""

Purpose: Extract audio features from transcript.

Parameters: – self: Instance reference – audio: Parameter for audio

AudioTextFusion (Line 86)

Description: Fuses audio features with text understanding.

Methods:

__init__ (Line 89)
def __init__(self, llm):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – llm: Parameter for llm

fuse (Line 92)
def fuse(self, audio_features, text_query):
    """Create unified audio-text understanding."""

Purpose: Create unified audio-text understanding.

Parameters: – self: Instance reference – audio_features: Parameter for audio_features – text_query: Parameter for text_query

AudioTextPattern (Line 123)

Description: Audio-Text Pattern – Processes combined audio and textual inputs.

Enables speech understanding with emotion and prosody awareness, suitable for voice assistants and conversational AI.

Methods:

__init__ (Line 131)
def __init__(self, model_name):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – model_name: LLM model to use (e.g., “gpt-4”)

process (Line 137)
def process(self, audio, text_query):
    """Process audio with optional text query."""

Purpose: Process audio with optional text query.

Parameters: – self: Instance reference – audio: Parameter for audio – text_query: Parameter for text_query

run (Line 175)
def run(self, speech_transcript, duration, text_query):
    """Main execution method."""

Purpose: Main execution method.

Parameters: – self: Instance reference – speech_transcript: Parameter for speech_transcript – duration: Parameter for duration – text_query: Parameter for text_query

Design Patterns Used

  • Strategy Pattern: Different execution strategies for various scenarios
  • State Pattern: Managing different states in the workflow
  • Chain of Responsibility: Sequential processing through workflow nodes
  • Observer Pattern: Tracking and monitoring pattern execution
  • Factory Pattern: Creating instances with proper configuration

Usage Example

# Initialize the pattern
pattern = AudioFeatureType(name="example", model_name="gpt-4")

# Configure and execute
result = pattern.execute(data={"key": "value"})

# Check results
print(result)

Key Concepts

  • LLM Integration: Uses LangChain/OpenAI for intelligent decision-making
  • Workflow Orchestration: LangGraph manages execution flow through nodes
  • State Management: TypedDict defines workflow state structure
  • Error Handling: Comprehensive error handling and fallback mechanisms
  • Extensibility: Designed for easy customization and extension

Extension Points

This pattern can be extended by: 1. Adding custom workflow nodes 2. Implementing additional validation logic 3. Integrating with external services 4. Customizing LLM prompts and responses 5. Adding monitoring and logging capabilities

Comments

Leave a Reply

Your email address will not be published. Required fields are marked *

More posts