"""
Pattern 127: Semantic Search & Retrieval

Description:
    Retrieves information based on semantic similarity rather than keyword matching,
    using embeddings and vector databases for intelligent information retrieval.

Components:
    - Text embeddings
    - Vector similarity search
    - Semantic ranking
    - Hybrid retrieval (keyword + semantic)

Use Cases:
    - Intelligent document search
    - Question answering systems
    - Recommendation systems

LangChain Implementation:
    Uses LangChain embeddings, vector stores, and retrievers for semantic search.
"""

import os
from typing import List, Dict, Any, Optional
from dotenv import load_dotenv

from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_core.documents import Document
from langchain_community.vectorstores import Chroma
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_core.runnables import RunnablePassthrough
from langchain.retrievers import EnsembleRetriever, ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor
from langchain_community.retrievers import BM25Retriever

load_dotenv()


class SemanticSearchAgent:
    """Agent for semantic search and retrieval"""
    
    def __init__(self, model: str = "gpt-4o-mini", temperature: float = 0.0):
        self.llm = ChatOpenAI(model=model, temperature=temperature)
        self.embeddings = OpenAIEmbeddings()
        self.vectorstore: Optional[Chroma] = None
        self.documents: List[Document] = []
        
        # Text splitter for chunking
        self.text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=500,
            chunk_overlap=50,
            length_function=len
        )
        
        # QA prompt
        self.qa_prompt = ChatPromptTemplate.from_messages([
            ("system", """You are a helpful assistant that answers questions based on the provided context.
Use only the information from the context to answer. If you cannot answer from the context, say so.
Always cite which part of the context you used."""),
            ("user", """Context:
{context}

Question: {question}

Answer:""")
        ])
    
    def load_documents(self, documents: List[str], metadatas: Optional[List[Dict]] = None):
        """Load documents into the system"""
        print(f"\n📚 Loading {len(documents)} documents...")
        
        # Create Document objects
        if metadatas:
            self.documents = [
                Document(page_content=doc, metadata=meta)
                for doc, meta in zip(documents, metadatas)
            ]
        else:
            self.documents = [
                Document(page_content=doc, metadata={"id": i})
                for i, doc in enumerate(documents)
            ]
        
        # Split documents
        split_docs = self.text_splitter.split_documents(self.documents)
        print(f"   Split into {len(split_docs)} chunks")
        
        # Create vector store
        self.vectorstore = Chroma.from_documents(
            documents=split_docs,
            embedding=self.embeddings,
            collection_name="semantic_search"
        )
        
        print(f"   ✓ Vectorstore created with {len(split_docs)} embeddings")
    
    def semantic_search(self, query: str, k: int = 3) -> List[Document]:
        """Pure semantic search using vector similarity"""
        print(f"\n🔍 Semantic search for: '{query}'")
        
        if not self.vectorstore:
            print("   ❌ No documents loaded!")
            return []
        
        # Retrieve similar documents
        results = self.vectorstore.similarity_search_with_score(query, k=k)
        
        print(f"   Found {len(results)} results:\n")
        for i, (doc, score) in enumerate(results, 1):
            print(f"   {i}. Score: {score:.4f}")
            print(f"      {doc.page_content[:100]}...\n")
        
        return [doc for doc, _ in results]
    
    def hybrid_search(self, query: str, k: int = 3) -> List[Document]:
        """Hybrid search combining semantic and keyword matching"""
        print(f"\n🔍 Hybrid search for: '{query}'")
        
        if not self.vectorstore or not self.documents:
            print("   ❌ No documents loaded!")
            return []
        
        # Create semantic retriever
        semantic_retriever = self.vectorstore.as_retriever(search_kwargs={"k": k})
        
        # Create keyword retriever (BM25)
        bm25_retriever = BM25Retriever.from_documents(self.documents)
        bm25_retriever.k = k
        
        # Combine retrievers
        ensemble_retriever = EnsembleRetriever(
            retrievers=[semantic_retriever, bm25_retriever],
            weights=[0.7, 0.3]  # Favor semantic
        )
        
        results = ensemble_retriever.get_relevant_documents(query)
        
        print(f"   Found {len(results)} results (semantic + keyword)\n")
        for i, doc in enumerate(results[:k], 1):
            print(f"   {i}. {doc.page_content[:100]}...\n")
        
        return results[:k]
    
    def search_with_reranking(self, query: str, k: int = 3) -> List[Document]:
        """Search with LLM-based reranking/compression"""
        print(f"\n🔍 Search with reranking for: '{query}'")
        
        if not self.vectorstore:
            print("   ❌ No documents loaded!")
            return []
        
        # Base retriever
        base_retriever = self.vectorstore.as_retriever(search_kwargs={"k": k * 2})
        
        # Compressor for reranking
        compressor = LLMChainExtractor.from_llm(self.llm)
        
        # Compression retriever
        compression_retriever = ContextualCompressionRetriever(
            base_compressor=compressor,
            base_retriever=base_retriever
        )
        
        results = compression_retriever.get_relevant_documents(query)
        
        print(f"   Found and reranked {len(results)} results\n")
        for i, doc in enumerate(results[:k], 1):
            print(f"   {i}. {doc.page_content[:150]}...\n")
        
        return results[:k]
    
    def answer_question(self, question: str, search_method: str = "semantic") -> str:
        """Answer question using retrieved context"""
        print(f"\n💬 Answering question: '{question}'")
        print(f"   Using {search_method} search")
        
        # Retrieve context
        if search_method == "semantic":
            context_docs = self.semantic_search(question, k=3)
        elif search_method == "hybrid":
            context_docs = self.hybrid_search(question, k=3)
        else:  # reranking
            context_docs = self.search_with_reranking(question, k=3)
        
        if not context_docs:
            return "No relevant context found."
        
        # Combine context
        context = "\n\n".join([doc.page_content for doc in context_docs])
        
        # Generate answer
        chain = self.qa_prompt | self.llm | StrOutputParser()
        answer = chain.invoke({"context": context, "question": question})
        
        print(f"\n📝 Answer:\n{answer}")
        
        return answer
    
    def find_similar_documents(self, reference_text: str, k: int = 3) -> List[Document]:
        """Find documents similar to reference text"""
        print(f"\n🔍 Finding similar documents to reference text...")
        
        if not self.vectorstore:
            print("   ❌ No documents loaded!")
            return []
        
        results = self.vectorstore.similarity_search(reference_text, k=k)
        
        print(f"   Found {len(results)} similar documents:\n")
        for i, doc in enumerate(results, 1):
            print(f"   {i}. {doc.page_content[:100]}...")
            if 'id' in doc.metadata:
                print(f"      (Document ID: {doc.metadata['id']})\n")
        
        return results
    
    def get_statistics(self) -> Dict[str, Any]:
        """Get search system statistics"""
        return {
            "total_documents": len(self.documents),
            "vectorstore_loaded": self.vectorstore is not None,
            "embedding_model": "text-embedding-ada-002"
        }


def demonstrate_semantic_search():
    """Demonstrate semantic search and retrieval"""
    print("=" * 80)
    print("Pattern 127: Semantic Search & Retrieval")
    print("=" * 80)
    
    agent = SemanticSearchAgent()
    
    # Example 1: Load documents
    print("\n" + "=" * 80)
    print("EXAMPLE 1: Loading Documents")
    print("=" * 80)
    
    documents = [
        """Python is a high-level, interpreted programming language known for its simplicity 
        and readability. Created by Guido van Rossum in 1991, Python emphasizes code 
        readability and allows programmers to express concepts in fewer lines of code.""",
        
        """Machine learning is a subset of artificial intelligence that enables systems to 
        learn and improve from experience without being explicitly programmed. It focuses 
        on developing computer programs that can access data and learn from it.""",
        
        """Deep learning is a subset of machine learning that uses neural networks with 
        multiple layers. These deep neural networks attempt to simulate the behavior of 
        the human brain, allowing it to learn from large amounts of data.""",
        
        """Natural Language Processing (NLP) is a branch of AI that helps computers understand, 
        interpret, and manipulate human language. NLP draws from many disciplines, including 
        computer science and computational linguistics.""",
        
        """Computer vision is a field of artificial intelligence that trains computers to 
        interpret and understand the visual world. Using digital images and deep learning 
        models, machines can accurately identify and classify objects.""",
        
        """The Transformer architecture, introduced in the paper 'Attention is All You Need', 
        revolutionized NLP. It relies entirely on attention mechanisms, dispensing with 
        recurrence and convolutions entirely."""
    ]
    
    metadatas = [
        {"topic": "programming", "language": "python"},
        {"topic": "AI", "subtopic": "machine_learning"},
        {"topic": "AI", "subtopic": "deep_learning"},
        {"topic": "AI", "subtopic": "NLP"},
        {"topic": "AI", "subtopic": "computer_vision"},
        {"topic": "AI", "subtopic": "transformers"}
    ]
    
    agent.load_documents(documents, metadatas)
    
    # Example 2: Semantic search
    print("\n" + "=" * 80)
    print("EXAMPLE 2: Pure Semantic Search")
    print("=" * 80)
    
    queries = [
        "neural networks and learning from data",
        "understanding human language with computers",
        "programming language for beginners"
    ]
    
    for query in queries:
        agent.semantic_search(query, k=2)
    
    # Example 3: Hybrid search
    print("\n" + "=" * 80)
    print("EXAMPLE 3: Hybrid Search (Semantic + Keyword)")
    print("=" * 80)
    
    agent.hybrid_search("deep neural networks", k=2)
    
    # Example 4: Search with reranking
    print("\n" + "=" * 80)
    print("EXAMPLE 4: Search with LLM Reranking")
    print("=" * 80)
    
    agent.search_with_reranking("AI techniques for processing text", k=2)
    
    # Example 5: Question answering with different search methods
    print("\n" + "=" * 80)
    print("EXAMPLE 5: Question Answering")
    print("=" * 80)
    
    questions = [
        "What is the Transformer architecture?",
        "Who created Python and when?",
        "What is the relationship between deep learning and machine learning?"
    ]
    
    methods = ["semantic", "hybrid", "reranking"]
    
    for question in questions[:1]:  # Just first question with all methods
        print(f"\nQuestion: {question}\n")
        for method in methods:
            print(f"\n--- Method: {method} ---")
            agent.answer_question(question, search_method=method)
    
    # Example 6: Find similar documents
    print("\n" + "=" * 80)
    print("EXAMPLE 6: Finding Similar Documents")
    print("=" * 80)
    
    reference = "Neural networks that can process and generate human language"
    agent.find_similar_documents(reference, k=3)
    
    # Summary
    print("\n" + "=" * 80)
    print("SUMMARY")
    print("=" * 80)
    print("""
Semantic Search & Retrieval Pattern:
- Finds information based on meaning, not just keywords
- Uses embeddings to capture semantic similarity
- Multiple search strategies available
- Enables intelligent question answering

Search Methods:
1. Pure Semantic: Vector similarity only
2. Hybrid: Combines semantic + keyword (BM25)
3. With Reranking: LLM compresses and reranks results

Key Benefits:
✓ Understanding user intent
✓ Finding conceptually similar content
✓ Better than keyword matching
✓ Handles synonyms and paraphrases
✓ Improved retrieval accuracy

Use Cases:
• Document search systems
• Question answering
• Recommendation engines
• Knowledge bases
• Research assistants
    """)


if __name__ == "__main__":
    demonstrate_semantic_search()

    

Agentic Design Pattern: Semantic Search Retrieval

Pattern 127: Semantic Search Retrieval – Line-by-Line Explanation

Overview

This document provides a detailed line-by-line explanation of the Semantic Search Retrieval pattern implementation.

---

Pattern Description

Pattern 127: Semantic Search & Retrieval

Description: Retrieves information based on semantic similarity rather than keyword matching, using embeddings and vector databases for intelligent information retrieval.

Components: – Text embeddings – Vector similarity search – Semantic ranking – Hybrid retrieval (keyword + semantic)

Use Cases: – Intelligent document search – Question answering systems – Recommendation systems

LangChain Implementation: Uses LangChain embeddings, vector stores, and retrievers for semantic search.

---

File Statistics

• Total Lines: 367
• Classes: 1
• Functions: 1
• Constants: 0

---

Module Documentation

"""
Pattern 127: Semantic Search & Retrieval

Description:
    Retrieves information based on semantic similarity rather than keyword matching,
    using embeddings and vector databases for intelligent information retrieval.

Components:
    - Text embeddings
    - Vector similarity search
    - Semantic ranking
    - Hybrid retrieval (keyword + semantic)

Use Cases:
    - Intelligent document search
    - Question answering systems
    - Recommendation systems

LangChain Implementation:
    Uses LangChain embeddings, vector stores, and retrievers for semantic search.
"""

Purpose: This module implements the Semantic Search Retrieval pattern, providing a structured approach to agent design and implementation.

---

Import Statements

Standard Library Imports

import langchain.retrievers.ContextualCompressionRetriever
import langchain.retrievers.EnsembleRetriever
import langchain.retrievers.document_compressors.LLMChainExtractor
import langchain_community.retrievers.BM25Retriever
import langchain_community.vectorstores.Chroma
import langchain_core.documents.Document
import langchain_core.output_parsers.StrOutputParser
import langchain_core.prompts.ChatPromptTemplate
import langchain_core.runnables.RunnablePassthrough
import os
import typing.Any
import typing.Dict
import typing.List
import typing.Optional

Purpose: Core Python functionality: – langchain.retrievers.ContextualCompressionRetriever: Regular expression operations – langchain.retrievers.EnsembleRetriever: Regular expression operations – langchain.retrievers.document_compressors.LLMChainExtractor: Regular expression operations – langchain_community.retrievers.BM25Retriever: Regular expression operations – langchain_community.vectorstores.Chroma: Regular expression operations – langchain_core.documents.Document: Regular expression operations – langchain_core.output_parsers.StrOutputParser: Regular expression operations – langchain_core.prompts.ChatPromptTemplate: Regular expression operations – langchain_core.runnables.RunnablePassthrough: Regular expression operations – os: Environment variable access and system operations – typing.Any: Type hints for better code documentation (List, Dict, Any, Optional) – typing.Dict: Type hints for better code documentation (List, Dict, Any, Optional) – typing.List: Type hints for better code documentation (List, Dict, Any, Optional) – typing.Optional: Type hints for better code documentation (List, Dict, Any, Optional)

LangChain Imports

from langchain.text_splitter.RecursiveCharacterTextSplitter
from langchain_openai.ChatOpenAI
from langchain_openai.OpenAIEmbeddings

Purpose: LangChain framework components: – LLM Interface: Wrapper for OpenAI models – Chains: Composable building blocks for complex workflows

Other Imports

from dotenv.load_dotenv

Purpose: Additional dependencies for specialized functionality.

---

Classes

Class: SemanticSearchAgent

Lines 41-225

Purpose:

Agent for semantic search and retrieval

Methods:

__init__(self, model, temperature)

Lines 44-68

Purpose: Initialize the class instance with required parameters.

Parameters: – model: Language model instance – temperature: Sampling temperature (0-1)

load_documents(self, documents, metadatas)

Lines 70-97

Purpose:

Load documents into the system

Parameters: – documents: Documents – metadatas: Metadatas

semantic_search(self, query, k)

Lines 99-115

Purpose:

Pure semantic search using vector similarity

Parameters: – query: Input query or question – k: K

Returns: <ast.Subscript object at 0x000001A06D62A390>

hybrid_search(self, query, k)

Lines 117-144

Purpose:

Hybrid search combining semantic and keyword matching

Parameters: – query: Input query or question – k: K

Returns: <ast.Subscript object at 0x000001A06D77C1D0>

search_with_reranking(self, query, k)

Lines 146-172

Purpose:

Search with LLM-based reranking/compression

Parameters: – query: Input query or question – k: K

Returns: <ast.Subscript object at 0x000001A06D7EE650>

answer_question(self, question, search_method)

Lines 174-199

Purpose:

Answer question using retrieved context

Parameters: – question: Question to answer – search_method: Search method

Returns: str

find_similar_documents(self, reference_text, k)

Lines 201-217

Purpose:

Find documents similar to reference text

Parameters: – reference_text: Reference text – k: K

Returns: <ast.Subscript object at 0x000001A06D7B3890>

get_statistics(self)

Lines 219-225

Purpose:

Get search system statistics

Parameters:

Returns: <ast.Subscript object at 0x000001A06D7B0050>

---

Functions

demonstrate_semantic_search()

Lines 228-362

Purpose:

Demonstrate semantic search and retrieval

---

Key Concepts Summary

Pattern Overview

The Semantic Search Retrieval pattern provides a structured approach to building AI agents with specific capabilities and behaviors. This implementation leverages LangChain’s composable components to create a robust and maintainable solution.

Core Components

1. Agent Class: Main implementation of the pattern logic
2. Prompt Engineering: Carefully crafted prompts to guide LLM behavior
3. Tool Integration: Optional tools for extended functionality
4. Memory Management: State handling across interactions
5. Output Parsing: Structured response processing

When to Use

✅ Good for: – Scenarios requiring this specific pattern’s strengths – Applications needing structured agent behavior – Systems where explainability is important – Production environments requiring reliability

❌ Not ideal for: – Simple single-prompt tasks – Scenarios requiring different pattern characteristics – Resource-constrained environments (if computationally intensive)

---

Best Practices Demonstrated

1. Environment Configuration

• ✅ Use .env files for API keys
• ✅ Never hardcode credentials
• ✅ Validate environment variables on startup

2. Error Handling

• ✅ Graceful degradation on failures
• ✅ Informative error messages
• ✅ Logging for debugging

3. Prompt Engineering

• ✅ Clear, specific instructions
• ✅ Structured output formats
• ✅ Few-shot examples when beneficial

4. Code Organization

• ✅ Separation of concerns (class-based design)
• ✅ Type hints for clarity
• ✅ Docstrings for documentation

5. Performance Optimization

• ✅ Appropriate temperature settings
• ✅ Token budget management
• ✅ Caching where applicable

---

LangChain Components Used

• ChatOpenAI: OpenAI model interface
• Chains: Sequential component composition

Why LangChain? – Composable abstractions for LLM applications – Rich ecosystem of integrations – Production-ready patterns – Active community and development

---

Execution Flow

1. Initialize agent with configuration
   ↓
2. Load environment and dependencies
   ↓
3. Prepare prompt with input data
   ↓
4. Invoke LLM through LangChain
   ↓
5. Parse and validate response
   ↓
6. Return formatted result

Key Decision Points: – Model selection (temperature, model type) – Prompt structure (few-shot vs zero-shot) – Error handling strategy – Output format validation

---

Performance Considerations

Computational Cost

• LLM Calls: Primary cost factor (based on tokens)
• Latency: Depends on model and input size
• Memory: Minimal for most patterns

Optimization Strategies

1. Prompt Optimization: Minimize tokens while maintaining clarity
2. Caching: Cache repeated queries/responses
3. Model Selection: Use smaller models when appropriate
4. Batch Processing: Group similar requests
5. Streaming: For long responses, stream tokens

Cost Estimates (Approximate)

• GPT-3.5-turbo: ~$0.002 per request
• GPT-4: ~$0.03 per request
• GPT-4-turbo: ~$0.01 per request

Actual costs vary based on input/output token counts

---

Common Pitfalls to Avoid

1. Prompt Design Issues

❌ Vague or ambiguous instructions
✅ Clear, specific, testable prompts

2. Error Handling

❌ Ignoring API errors or rate limits
✅ Implement retry logic and graceful degradation

3. Token Management

❌ Exceeding model token limits
✅ Truncate inputs, chunk processing

4. Security

❌ Exposing API keys in code
✅ Use environment variables

5. Testing

❌ Only testing happy paths
✅ Test edge cases, failures, malformed inputs

6. Cost Management

❌ Unmonitored API usage
✅ Implement budgets, alerts, and monitoring

---

Further Improvements

Production Enhancements

1. Monitoring: Add logging, metrics, tracing
2. Validation: Input/output schema validation
3. Rate Limiting: Implement request throttling
4. Caching: Redis or in-memory cache for responses
5. Async Support: Non-blocking operations for scalability

Feature Extensions

1. Multi-Model Support: Switch between providers
2. Fallback Models: Graceful degradation strategy
3. Custom Tools: Domain-specific function calling
4. Advanced Memory: Vector stores for long-term memory
5. Evaluation: Automated testing and benchmarking

Code Quality

1. Testing: Unit tests, integration tests
2. Documentation: API docs, usage examples
3. Type Safety: Stricter type hints
4. Configuration: Externalized config files

---

Related Patterns

• Pattern 126
• Pattern 128

Pattern Families: – Reasoning Patterns (001-011) – Multi-Agent Patterns (012-019) – Tool Use Patterns (020-025) – Memory Patterns (026-031) – And many more…

---

Summary

The Semantic Search Retrieval pattern provides a powerful approach to building AI agents with specific capabilities. This implementation demonstrates:

✅ Key Strengths: – Clean, maintainable code structure – LangChain integration for composability – Type hints and documentation – Production-ready patterns

🎯 Best Use Cases: – Applications requiring this pattern’s characteristics – Systems needing structured agent behavior – Production environments requiring reliability

📚 Learning Resources: – LangChain Documentation: https://python.langchain.com/ – Pattern Documentation: See related patterns – OpenAI API Reference: https://platform.openai.com/docs/

---

File: 127_semantic_search_retrieval.py
Total Lines: 367
Pattern Category: AI Agent Design Pattern #127

---

Generated from actual implementation code
For educational and reference purposes