aishastr.live

Agentic Design Pattern: Imitation Learning

Written by

admin

in


    """
Pattern 117: Imitation Learning

Description:
    Agent learns by observing and imitating expert behavior through
    behavioral cloning and inverse reinforcement learning approaches.

Components:
    - Expert demonstrations
    - Behavioral cloning
    - Policy learning

Use Cases:
    - Robotics
    - Complex skill acquisition
    - Learning from examples

LangChain Implementation:
    Uses few-shot learning and demonstration-based prompting to imitate expert behavior.
"""

import os
from typing import List, Dict, Any
from dotenv import load_dotenv

from langchain_core.prompts import ChatPromptTemplate, FewShotChatMessagePromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_openai import ChatOpenAI

load_dotenv()


class ImitationLearningAgent:
    """Agent that learns by observing and imitating expert demonstrations."""
    
    def __init__(self, model_name: str = "gpt-4"):
        self.llm = ChatOpenAI(model=model_name, temperature=0.3)
        self.expert_demonstrations = []
        self.learned_policy = None
        
    def observe_expert(self, situation: str, expert_action: str, reasoning: str):
        """Observe and record expert demonstration."""
        demonstration = {
            "situation": situation,
            "action": expert_action,
            "reasoning": reasoning
        }
        self.expert_demonstrations.append(demonstration)
        print(f"Observed demonstration {len(self.expert_demonstrations)}:")
        print(f"  Situation: {situation}")
        print(f"  Action: {expert_action}")
        print(f"  Reasoning: {reasoning}")
        print()
        
    def learn_from_demonstrations(self):
        """Learn policy from all observed demonstrations."""
        learning_prompt = ChatPromptTemplate.from_messages([
            ("system", "Analyze these expert demonstrations and extract the underlying policy, patterns, and decision-making strategy."),
            ("user", """Expert Demonstrations:
{demonstrations}

Extract:
1. Common patterns in decision-making
2. Key factors considered
3. General policy rules
4. Decision strategy""")
        ])
        
        demos_text = "\n\n".join([
            f"Demo {i+1}:\nSituation: {d['situation']}\nAction: {d['action']}\nReasoning: {d['reasoning']}"
            for i, d in enumerate(self.expert_demonstrations)
        ])
        
        chain = learning_prompt | self.llm | StrOutputParser()
        self.learned_policy = chain.invoke({"demonstrations": demos_text})
        
        print("=== Learned Policy ===")
        print(self.learned_policy)
        print()
        
    def imitate_expert(self, new_situation: str) -> Dict[str, str]:
        """Apply learned policy to a new situation."""
        if not self.learned_policy:
            return {"error": "No policy learned yet. Need more demonstrations."}
        
        # Create few-shot prompt with demonstrations
        example_template = """Situation: {situation}
Expert Action: {action}
Reasoning: {reasoning}"""
        
        few_shot_prompt = FewShotChatMessagePromptTemplate(
            example_prompt=ChatPromptTemplate.from_messages([
                ("user", "Situation: {situation}"),
                ("assistant", "Action: {action}\nReasoning: {reasoning}")
            ]),
            examples=self.expert_demonstrations,
        )
        
        full_prompt = ChatPromptTemplate.from_messages([
            ("system", """You are imitating an expert's decision-making style.

Learned Policy:
{learned_policy}

Use the learned policy and demonstrated patterns to decide actions for new situations."""),
            few_shot_prompt,
            ("user", "Situation: {new_situation}\n\nWhat action should be taken? Provide action and reasoning.")
        ])
        
        chain = full_prompt | self.llm | StrOutputParser()
        response = chain.invoke({
            "learned_policy": self.learned_policy,
            "new_situation": new_situation
        })
        
        # Parse response
        lines = response.split('\n')
        action = ""
        reasoning = ""
        
        for line in lines:
            if line.startswith("Action:"):
                action = line.replace("Action:", "").strip()
            elif line.startswith("Reasoning:"):
                reasoning = line.replace("Reasoning:", "").strip()
            elif action and not reasoning:
                reasoning = line.strip()
        
        return {
            "situation": new_situation,
            "action": action or response,
            "reasoning": reasoning or "Based on learned policy"
        }
    
    def evaluate_imitation(self, test_situation: str, expected_action: str) -> bool:
        """Evaluate how well the agent imitates expert behavior."""
        result = self.imitate_expert(test_situation)
        
        print(f"Test Situation: {test_situation}")
        print(f"Expected Action: {expected_action}")
        print(f"Agent Action: {result['action']}")
        print(f"Reasoning: {result['reasoning']}")
        
        # Simple similarity check
        similarity = expected_action.lower() in result['action'].lower()
        print(f"Match: {'✓' if similarity else '✗'}")
        print()
        
        return similarity


def demonstrate_imitation_learning():
    """Demonstrate imitation learning pattern."""
    print("=== Imitation Learning Pattern ===\n")
    
    agent = ImitationLearningAgent()
    
    # Scenario: Learning customer service expert behavior
    print("1. Observing Expert Demonstrations")
    print("-" * 50)
    
    # Demonstration 1: Angry customer
    agent.observe_expert(
        situation="Customer is angry about a delayed delivery",
        expert_action="Apologize sincerely, explain the situation, offer compensation (discount or free shipping on next order)",
        reasoning="Acknowledge emotions first, show empathy, provide solution to restore trust"
    )
    
    # Demonstration 2: Confused customer
    agent.observe_expert(
        situation="Customer is confused about how to use a product feature",
        expert_action="Ask clarifying questions, provide step-by-step guidance, offer visual aids or video tutorial",
        reasoning="Understand specific confusion point, break down into simple steps, provide multiple learning formats"
    )
    
    # Demonstration 3: Billing inquiry
    agent.observe_expert(
        situation="Customer questions an unexpected charge on their bill",
        expert_action="Review the charge details, explain clearly what it's for, provide itemized breakdown if needed",
        reasoning="Be transparent about charges, ensure customer understands, address concerns directly"
    )
    
    # Demonstration 4: Feature request
    agent.observe_expert(
        situation="Customer requests a feature that doesn't exist",
        expert_action="Thank them for feedback, explain current capabilities, document request for product team, suggest workaround if possible",
        reasoning="Value customer input, manage expectations, show commitment to improvement, help immediately if possible"
    )
    
    print("✓ Observed 4 expert demonstrations\n")
    
    # Learn from demonstrations
    print("2. Learning Policy from Demonstrations")
    print("-" * 50)
    agent.learn_from_demonstrations()
    
    # Test imitation on new situations
    print("3. Testing Imitation on New Situations")
    print("-" * 50)
    
    # Test 1: Product defect
    print("Test 1: Product Defect Situation")
    result = agent.imitate_expert("Customer reports a defective product that stopped working after one week")
    print(f"Situation: {result['situation']}")
    print(f"Action: {result['action']}")
    print(f"Reasoning: {result['reasoning']}")
    print()
    
    # Test 2: Refund request
    print("Test 2: Refund Request Situation")
    result = agent.imitate_expert("Customer wants a refund for a product they bought 2 months ago and barely used")
    print(f"Situation: {result['situation']}")
    print(f"Action: {result['action']}")
    print(f"Reasoning: {result['reasoning']}")
    print()
    
    # Test 3: Complex technical issue
    print("Test 3: Technical Issue Situation")
    result = agent.imitate_expert("Customer experiencing intermittent connectivity issues with the app")
    print(f"Situation: {result['situation']}")
    print(f"Action: {result['action']}")
    print(f"Reasoning: {result['reasoning']}")
    print()
    
    # Evaluation
    print("4. Evaluating Imitation Quality")
    print("-" * 50)
    agent.evaluate_imitation(
        "Customer complaint about rude staff member",
        "Apologize for experience, escalate to manager, ensure follow-up"
    )
    
    print("=== Summary ===")
    print("Imitation learning demonstrated with:")
    print("- Expert demonstration observation")
    print("- Policy learning from demonstrations")
    print("- Behavioral cloning for new situations")
    print("- Few-shot learning approach")
    print("- Pattern extraction from expert behavior")


if __name__ == "__main__":
    demonstrate_imitation_learning()

Agentic Design Pattern: Imitation Learning

Pattern 117: Imitation Learning – Line-by-Line Explanation

Overview

This document provides a detailed line-by-line explanation of the Imitation Learning pattern implementation.

Pattern Description

Pattern 117: Imitation Learning

Description: Agent learns by observing and imitating expert behavior through behavioral cloning and inverse reinforcement learning approaches.

Components: – Expert demonstrations – Behavioral cloning – Policy learning

Use Cases: – Robotics – Complex skill acquisition – Learning from examples

LangChain Implementation: Uses few-shot learning and demonstration-based prompting to imitate expert behavior.

File Statistics

  • Total Lines: 244
  • Classes: 1
  • Functions: 1
  • Constants: 0

Module Documentation

"""
Pattern 117: Imitation Learning

Description:
    Agent learns by observing and imitating expert behavior through
    behavioral cloning and inverse reinforcement learning approaches.

Components:
    - Expert demonstrations
    - Behavioral cloning
    - Policy learning

Use Cases:
    - Robotics
    - Complex skill acquisition
    - Learning from examples

LangChain Implementation:
    Uses few-shot learning and demonstration-based prompting to imitate expert behavior.
"""

Purpose: This module implements the Imitation Learning pattern, providing a structured approach to agent design and implementation.

Import Statements

Standard Library Imports

import langchain_core.output_parsers.StrOutputParser
import langchain_core.prompts.ChatPromptTemplate
import langchain_core.prompts.FewShotChatMessagePromptTemplate
import os
import typing.Any
import typing.Dict
import typing.List

Purpose: Core Python functionality: – langchain_core.output_parsers.StrOutputParser: Regular expression operations – langchain_core.prompts.ChatPromptTemplate: Regular expression operations – langchain_core.prompts.FewShotChatMessagePromptTemplate: Regular expression operations – os: Environment variable access and system operations – typing.Any: Type hints for better code documentation (List, Dict, Any, Optional) – typing.Dict: Type hints for better code documentation (List, Dict, Any, Optional) – typing.List: Type hints for better code documentation (List, Dict, Any, Optional)

LangChain Imports

from langchain_openai.ChatOpenAI

Purpose: LangChain framework components: – LLM Interface: Wrapper for OpenAI models – Chains: Composable building blocks for complex workflows

Other Imports

from dotenv.load_dotenv

Purpose: Additional dependencies for specialized functionality.

Classes

Class: ImitationLearningAgent

Lines 33-149

Purpose:

Agent that learns by observing and imitating expert demonstrations.

Methods:

__init__(self, model_name)

Lines 36-39

Purpose: Initialize the class instance with required parameters.

Parameters: – model_name: Language model instance

observe_expert(self, situation, expert_action, reasoning)

Lines 41-53

Purpose:

Observe and record expert demonstration.

Parameters: – situation: Situation – expert_action: Expert action – reasoning: Reasoning

learn_from_demonstrations(self)

Lines 55-79

Purpose:

Learn policy from all observed demonstrations.

Parameters:

imitate_expert(self, new_situation)

Lines 81-133

Purpose:

Apply learned policy to a new situation.

Parameters: – new_situation: New situation

Returns: <ast.Subscript object at 0x000001A06D7F4090>

evaluate_imitation(self, test_situation, expected_action)

Lines 135-149

Purpose:

Evaluate how well the agent imitates expert behavior.

Parameters: – test_situation: Test situation – expected_action: Expected action

Returns: bool

Functions

demonstrate_imitation_learning()

Lines 152-239

Purpose:

Demonstrate imitation learning pattern.

Key Concepts Summary

Pattern Overview

The Imitation Learning pattern provides a structured approach to building AI agents with specific capabilities and behaviors. This implementation leverages LangChain’s composable components to create a robust and maintainable solution.

Core Components

  1. Agent Class: Main implementation of the pattern logic
  2. Prompt Engineering: Carefully crafted prompts to guide LLM behavior
  3. Tool Integration: Optional tools for extended functionality
  4. Memory Management: State handling across interactions
  5. Output Parsing: Structured response processing

When to Use

✅ Good for: – Scenarios requiring this specific pattern’s strengths – Applications needing structured agent behavior – Systems where explainability is important – Production environments requiring reliability

❌ Not ideal for: – Simple single-prompt tasks – Scenarios requiring different pattern characteristics – Resource-constrained environments (if computationally intensive)

Best Practices Demonstrated

1. Environment Configuration

  • ✅ Use .env files for API keys
  • ✅ Never hardcode credentials
  • ✅ Validate environment variables on startup

2. Error Handling

  • ✅ Graceful degradation on failures
  • ✅ Informative error messages
  • ✅ Logging for debugging

3. Prompt Engineering

  • ✅ Clear, specific instructions
  • ✅ Structured output formats
  • ✅ Few-shot examples when beneficial

4. Code Organization

  • ✅ Separation of concerns (class-based design)
  • ✅ Type hints for clarity
  • ✅ Docstrings for documentation

5. Performance Optimization

  • ✅ Appropriate temperature settings
  • ✅ Token budget management
  • ✅ Caching where applicable

LangChain Components Used

  • ChatOpenAI: OpenAI model interface
  • Chains: Sequential component composition

Why LangChain? – Composable abstractions for LLM applications – Rich ecosystem of integrations – Production-ready patterns – Active community and development

Execution Flow

1. Initialize agent with configuration
   ↓
2. Load environment and dependencies
   ↓
3. Prepare prompt with input data
   ↓
4. Invoke LLM through LangChain
   ↓
5. Parse and validate response
   ↓
6. Return formatted result

Key Decision Points: – Model selection (temperature, model type) – Prompt structure (few-shot vs zero-shot) – Error handling strategy – Output format validation

Performance Considerations

Computational Cost

  • LLM Calls: Primary cost factor (based on tokens)
  • Latency: Depends on model and input size
  • Memory: Minimal for most patterns

Optimization Strategies

  1. Prompt Optimization: Minimize tokens while maintaining clarity
  2. Caching: Cache repeated queries/responses
  3. Model Selection: Use smaller models when appropriate
  4. Batch Processing: Group similar requests
  5. Streaming: For long responses, stream tokens

Cost Estimates (Approximate)

  • GPT-3.5-turbo: ~$0.002 per request
  • GPT-4: ~$0.03 per request
  • GPT-4-turbo: ~$0.01 per request

Actual costs vary based on input/output token counts

Common Pitfalls to Avoid

1. Prompt Design Issues

❌ Vague or ambiguous instructions
✅ Clear, specific, testable prompts

2. Error Handling

❌ Ignoring API errors or rate limits
✅ Implement retry logic and graceful degradation

3. Token Management

❌ Exceeding model token limits
✅ Truncate inputs, chunk processing

4. Security

❌ Exposing API keys in code
✅ Use environment variables

5. Testing

❌ Only testing happy paths
✅ Test edge cases, failures, malformed inputs

6. Cost Management

❌ Unmonitored API usage
✅ Implement budgets, alerts, and monitoring

Further Improvements

Production Enhancements

  1. Monitoring: Add logging, metrics, tracing
  2. Validation: Input/output schema validation
  3. Rate Limiting: Implement request throttling
  4. Caching: Redis or in-memory cache for responses
  5. Async Support: Non-blocking operations for scalability

Feature Extensions

  1. Multi-Model Support: Switch between providers
  2. Fallback Models: Graceful degradation strategy
  3. Custom Tools: Domain-specific function calling
  4. Advanced Memory: Vector stores for long-term memory
  5. Evaluation: Automated testing and benchmarking

Code Quality

  1. Testing: Unit tests, integration tests
  2. Documentation: API docs, usage examples
  3. Type Safety: Stricter type hints
  4. Configuration: Externalized config files

Related Patterns

  • Pattern 116
  • Pattern 118

Pattern Families: – Reasoning Patterns (001-011) – Multi-Agent Patterns (012-019) – Tool Use Patterns (020-025) – Memory Patterns (026-031) – And many more…

Summary

The Imitation Learning pattern provides a powerful approach to building AI agents with specific capabilities. This implementation demonstrates:

✅ Key Strengths: – Clean, maintainable code structure – LangChain integration for composability – Type hints and documentation – Production-ready patterns

🎯 Best Use Cases: – Applications requiring this pattern’s characteristics – Systems needing structured agent behavior – Production environments requiring reliability

📚 Learning Resources: – LangChain Documentation: https://python.langchain.com/ – Pattern Documentation: See related patterns – OpenAI API Reference: https://platform.openai.com/docs/

File: 117_imitation_learning.py
Total Lines: 244
Pattern Category: AI Agent Design Pattern #117

Generated from actual implementation code
For educational and reference purposes

Comments

Leave a Reply

Your email address will not be published. Required fields are marked *

More posts