aishastr.live

Agentic Architecture Pattern: Vision Language

Written by

admin

in


    """
Pattern 221: Vision-Language Pattern

Multi-modal agent that processes both visual and textual information to understand
and respond to combined image-text inputs. Integrates computer vision with language
models for unified understanding.
"""

from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI
from langgraph.graph import StateGraph, END
from typing_extensions import TypedDict


class ModalityType(Enum):
    """Types of input modalities."""
    VISION = "vision"
    LANGUAGE = "language"
    COMBINED = "combined"


@dataclass
class MultiModalInput:
    """Input combining vision and language."""
    image_description: str  # In real impl, would be image data/embeddings
    text: str
    modality: ModalityType
    timestamp: datetime = field(default_factory=datetime.now)


@dataclass
class VisionFeature:
    """Extracted visual features."""
    objects: List[str]
    scene: str
    attributes: Dict[str, Any]
    confidence: float = 0.0


@dataclass
class GroundedResponse:
    """Response grounded in both vision and language."""
    text_response: str
    visual_references: List[str]
    confidence: float
    reasoning: str


class VisionEncoder:
    """Simulates vision encoding (in real impl, would use vision model)."""
    
    def __init__(self, llm: ChatOpenAI):
        self.llm = llm
    
    def encode(self, image_description: str) -> VisionFeature:
        """Extract visual features from image description."""
        prompt = ChatPromptTemplate.from_messages([
            ("system", "You are a vision feature extractor. Extract objects, scene type, and key attributes."),
            ("human", "Image description: {desc}\n\nExtract: objects (list), scene (string), key attributes")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({"desc": image_description})
        content = str(response.content)
        
        # Parse response (simplified)
        objects = [obj.strip() for obj in content.split("Objects:")[1].split("Scene:")[0].strip().split(",") if obj.strip()][:5]
        scene = content.split("Scene:")[1].split("\n")[0].strip() if "Scene:" in content else "general"
        
        return VisionFeature(
            objects=objects,
            scene=scene,
            attributes={"description": image_description},
            confidence=0.85
        )


class LanguageEncoder:
    """Encodes language understanding."""
    
    def __init__(self, llm: ChatOpenAI):
        self.llm = llm
    
    def encode(self, text: str) -> Dict[str, Any]:
        """Extract linguistic features."""
        prompt = ChatPromptTemplate.from_messages([
            ("system", "Analyze this text for intent, entities, and key concepts."),
            ("human", "{text}")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({"text": text})
        
        return {
            "text": text,
            "analysis": str(response.content),
            "confidence": 0.9
        }


class VisionLanguageFusion:
    """Fuses vision and language representations."""
    
    def __init__(self, llm: ChatOpenAI):
        self.llm = llm
    
    def fuse(self, vision: VisionFeature, language: Dict[str, Any]) -> Dict[str, Any]:
        """Create unified representation."""
        prompt = ChatPromptTemplate.from_messages([
            ("system", "You are fusing visual and textual information into unified understanding."),
            ("human", """Visual info:
Objects: {objects}
Scene: {scene}

Textual info:
{text}

Provide a unified understanding that grounds the text in the visual context.""")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({
            "objects": ", ".join(vision.objects),
            "scene": vision.scene,
            "text": language["text"]
        })
        
        return {
            "unified": str(response.content),
            "visual_elements": vision.objects,
            "scene_context": vision.scene
        }


class VisionLanguagePattern:
    """
    Vision-Language Pattern - Processes combined visual and textual inputs.
    
    Integrates computer vision and NLP for multi-modal understanding,
    enabling agents to reason about images and text together.
    """
    
    def __init__(self, model_name: str = "gpt-4-vision-preview"):
        self.llm = ChatOpenAI(model=model_name, temperature=0.3)
        self.vision_encoder = VisionEncoder(self.llm)
        self.language_encoder = LanguageEncoder(self.llm)
        self.fusion = VisionLanguageFusion(self.llm)
        self.history: List[GroundedResponse] = []
    
    def process(self, input_data: MultiModalInput) -> GroundedResponse:
        """Process multi-modal input."""
        
        # Encode vision
        vision_features = self.vision_encoder.encode(input_data.image_description)
        
        # Encode language
        language_features = self.language_encoder.encode(input_data.text)
        
        # Fuse modalities
        fused = self.fusion.fuse(vision_features, language_features)
        
        # Generate grounded response
        prompt = ChatPromptTemplate.from_messages([
            ("system", """You are a vision-language agent. Generate responses that are grounded in both 
            the visual scene and the textual query. Reference specific visual elements."""),
            ("human", """Scene: {scene}
Objects visible: {objects}
Question/Text: {text}
Unified understanding: {unified}

Provide a detailed response that references specific visual elements.""")
        ])
        
        chain = prompt | self.llm
        response = chain.invoke({
            "scene": fused["scene_context"],
            "objects": ", ".join(fused["visual_elements"]),
            "text": input_data.text,
            "unified": fused["unified"]
        })
        
        grounded_response = GroundedResponse(
            text_response=str(response.content),
            visual_references=fused["visual_elements"],
            confidence=0.85,
            reasoning=fused["unified"]
        )
        
        self.history.append(grounded_response)
        return grounded_response
    
    def run(self, image_description: str, text_query: str) -> Dict[str, Any]:
        """Main execution method."""
        input_data = MultiModalInput(
            image_description=image_description,
            text=text_query,
            modality=ModalityType.COMBINED
        )
        
        response = self.process(input_data)
        
        return {
            "pattern": "Vision-Language Pattern",
            "response": response.text_response,
            "visual_references": response.visual_references,
            "confidence": response.confidence,
            "reasoning": response.reasoning
        }


# Example usage
if __name__ == "__main__":
    print("=== Vision-Language Pattern Demo ===\n")
    
    pattern = VisionLanguagePattern()
    
    # Example 1: Image understanding with question
    result1 = pattern.run(
        image_description="A sunny park with children playing on swings, a dog running, and people sitting on benches under trees",
        text_query="What activities are people doing in this scene?"
    )
    
    print(f"Query: {result1['response'][:200]}...")
    print(f"Visual References: {result1['visual_references']}")
    print(f"Confidence: {result1['confidence']}\n")
    
    # Example 2: Object localization
    result2 = pattern.run(
        image_description="A kitchen counter with a red apple, a glass of milk, a sandwich, and a newspaper",
        text_query="Where can I find something healthy to eat?"
    )
    
    print(f"Query: {result2['response'][:200]}...")
    print(f"Visual References: {result2['visual_references']}")
    print(f"Confidence: {result2['confidence']}\n")
    
    print(f"Total interactions: {len(pattern.history)}")

Agentic Architecture Pattern: Vision Language

Pattern 221: 221 Vision Language – Line-by-Line Explanation

Overview

Pattern 221: Vision-Language Pattern

Multi-modal agent that processes both visual and textual information to understand and respond to combined image-text inputs. Integrates computer vision with language models for unified understanding.

Code Structure Analysis

File Information

  • Pattern Number: 221
  • Pattern Name: 221 Vision Language
  • File Path: patterns\multi_modal\221_vision_language.py
  • Total Lines: 242

Detailed Code Explanation

Module Docstring

"""
Pattern 221: Vision-Language Pattern

Multi-modal agent that processes both visual and textual information to understand
and respond to combined image-text inputs. Integrates computer vision with language
models for unified understanding.
"""

Explanation: This module docstring describes the purpose and key features of the 221 Vision Language pattern.

Imports Section

from typing import Dict, Any, List, Optional
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum
from langchain_core.prompts import ChatPromptTemplate
from langchain_openai import ChatOpenAI
from langgraph.graph import StateGraph, END
from typing_extensions import TypedDict  # First 20 imports shown

Explanation: The module imports necessary dependencies: – from typing import Dict, Any, List, Optional: Type hints for better code documentation – from dataclasses import dataclass, field: Data class decorators for clean data structures – from datetime import datetime: Date and time utilities – from enum import Enum: Enumeration support for fixed sets of values – from langchain_core.prompts import ChatPromptTemplate: LangChain framework integration – from langchain_openai import ChatOpenAI: LangChain framework integration – from langgraph.graph import StateGraph, END: LangGraph for workflow orchestration – from typing_extensions import TypedDict: Type hints for better code documentation

Enumerations

ModalityType (Line 19)

class ModalityType(Enum):
    VISION
    LANGUAGE
    COMBINED

Purpose: Types of input modalities.

Values: – VISION: Represents vision – LANGUAGE: Represents language – COMBINED: Represents combined

Classes

ModalityType (Line 19)

Description: Types of input modalities.

MultiModalInput (Line 27)

Description: Input combining vision and language.

VisionFeature (Line 36)

Description: Extracted visual features.

GroundedResponse (Line 45)

Description: Response grounded in both vision and language.

VisionEncoder (Line 53)

Description: Simulates vision encoding (in real impl, would use vision model).

Methods:

__init__ (Line 56)
def __init__(self, llm):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – llm: Parameter for llm

encode (Line 59)
def encode(self, image_description):
    """Extract visual features from image description."""

Purpose: Extract visual features from image description.

Parameters: – self: Instance reference – image_description: Parameter for image_description

LanguageEncoder (Line 82)

Description: Encodes language understanding.

Methods:

__init__ (Line 85)
def __init__(self, llm):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – llm: Parameter for llm

encode (Line 88)
def encode(self, text):
    """Extract linguistic features."""

Purpose: Extract linguistic features.

Parameters: – self: Instance reference – text: Parameter for text

VisionLanguageFusion (Line 105)

Description: Fuses vision and language representations.

Methods:

__init__ (Line 108)
def __init__(self, llm):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – llm: Parameter for llm

fuse (Line 111)
def fuse(self, vision, language):
    """Create unified representation."""

Purpose: Create unified representation.

Parameters: – self: Instance reference – vision: Parameter for vision – language: Parameter for language

VisionLanguagePattern (Line 139)

Description: Vision-Language Pattern – Processes combined visual and textual inputs.

Integrates computer vision and NLP for multi-modal understanding, enabling agents to reason about images and text together.

Methods:

__init__ (Line 147)
def __init__(self, model_name):
    """No method docstring."""

Purpose: No method docstring.

Parameters: – self: Instance reference – model_name: LLM model to use (e.g., “gpt-4”)

process (Line 154)
def process(self, input_data):
    """Process multi-modal input."""

Purpose: Process multi-modal input.

Parameters: – self: Instance reference – input_data: Parameter for input_data

run (Line 196)
def run(self, image_description, text_query):
    """Main execution method."""

Purpose: Main execution method.

Parameters: – self: Instance reference – image_description: Parameter for image_description – text_query: Parameter for text_query

Design Patterns Used

  • Strategy Pattern: Different execution strategies for various scenarios
  • State Pattern: Managing different states in the workflow
  • Chain of Responsibility: Sequential processing through workflow nodes
  • Observer Pattern: Tracking and monitoring pattern execution
  • Factory Pattern: Creating instances with proper configuration

Usage Example

# Initialize the pattern
pattern = ModalityType(name="example", model_name="gpt-4")

# Configure and execute
result = pattern.execute(data={"key": "value"})

# Check results
print(result)

Key Concepts

  • LLM Integration: Uses LangChain/OpenAI for intelligent decision-making
  • Workflow Orchestration: LangGraph manages execution flow through nodes
  • State Management: TypedDict defines workflow state structure
  • Error Handling: Comprehensive error handling and fallback mechanisms
  • Extensibility: Designed for easy customization and extension

Extension Points

This pattern can be extended by: 1. Adding custom workflow nodes 2. Implementing additional validation logic 3. Integrating with external services 4. Customizing LLM prompts and responses 5. Adding monitoring and logging capabilities

Comments

Leave a Reply

Your email address will not be published. Required fields are marked *

More posts