"""
Pattern 152: Property-Based Testing
Description:
Implements property-based testing for AI agents by defining invariants
and properties that should hold across all inputs, then generating test
cases to verify these properties.
Components:
- Property Definitions: Invariants that should always hold
- Test Case Generators: Create diverse inputs
- Property Verifiers: Check if properties hold
- Hypothesis Engine: Generate test hypotheses
- Counterexample Finder: Identify violations
Use Cases:
- Testing agent robustness across input space
- Finding edge cases and failures
- Verifying consistency properties
- Ensuring output format compliance
- Testing deterministic vs non-deterministic behaviors
Benefits:
- Discovers unexpected edge cases
- More thorough than example-based testing
- Finds bugs traditional testing misses
- Documents expected behaviors as properties
- Scales testing coverage automatically
Trade-offs:
- Requires careful property definition
- Can be slower than unit tests
- May generate false positives
- Harder to debug failures
- Non-determinism complicates verification
LangChain Implementation:
Uses ChatOpenAI with property checking chains and systematic
test case generation.
"""
import os
import re
import time
from typing import List, Dict, Any, Optional, Callable, Tuple
from dataclasses import dataclass, field
from enum import Enum
from datetime import datetime
from collections import defaultdict
from dotenv import load_dotenv
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
load_dotenv()
class PropertyType(Enum):
"""Types of properties to test"""
INVARIANT = "invariant" # Must always hold
POSTCONDITION = "postcondition" # Output constraints
CONSISTENCY = "consistency" # Same input ā same output
MONOTONICITY = "monotonicity" # Ordering preservation
IDEMPOTENCE = "idempotence" # f(f(x)) = f(x)
COMMUTATIVE = "commutative" # Order independence
FORMAT = "format" # Output format compliance
@dataclass
class Property:
"""Defines a testable property"""
name: str
description: str
property_type: PropertyType
checker: Callable[[Any, Any], bool] # (input, output) -> bool
examples: List[Tuple[Any, bool]] = field(default_factory=list) # (input, expected_result)
def check(self, input_val: Any, output_val: Any) -> bool:
"""Check if property holds"""
return self.checker(input_val, output_val)
@dataclass
class TestResult:
"""Result of a property test"""
property_name: str
passed: bool
test_case: Any
output: Any
error: Optional[str] = None
execution_time: float = 0.0
timestamp: datetime = field(default_factory=datetime.now)
@dataclass
class PropertyTestReport:
"""Comprehensive test report"""
property_name: str
total_tests: int
passed: int
failed: int
counterexamples: List[Dict[str, Any]]
execution_time: float
@property
def pass_rate(self) -> float:
return self.passed / self.total_tests if self.total_tests > 0 else 0.0
class PropertyBasedTester:
"""
Agent that performs property-based testing on AI systems.
Generates test cases and verifies properties hold.
"""
def __init__(self, temperature: float = 0.7):
"""Initialize the property-based tester"""
self.llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=temperature)
self.consistent_llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0.0)
# Property registry
self.properties: Dict[str, Property] = {}
# Test case generator prompts
self.test_gen_prompt = ChatPromptTemplate.from_messages([
("system", """You are a test case generator. Generate diverse test cases for testing AI systems.
Property to test: {property_description}
Constraints: {constraints}
Generate test cases that:
1. Cover normal cases
2. Include edge cases
3. Test boundary conditions
4. Challenge the property
Return as a list, one test case per line."""),
("user", "Generate {count} test cases.")
])
self.agent_prompt = ChatPromptTemplate.from_messages([
("system", "You are a helpful AI assistant. Provide clear, accurate responses."),
("user", "{query}")
])
# Register default properties
self._register_default_properties()
def _register_default_properties(self):
"""Register commonly used properties"""
# 1. Non-empty response
self.register_property(Property(
name="non_empty_response",
description="Agent must return non-empty response",
property_type=PropertyType.POSTCONDITION,
checker=lambda inp, out: out is not None and len(str(out).strip()) > 0
))
# 2. Response length reasonable
self.register_property(Property(
name="reasonable_length",
description="Response length should be reasonable (not too short or long)",
property_type=PropertyType.POSTCONDITION,
checker=lambda inp, out: 10 < len(str(out)) len(str(inp)) * 0.5
))
# 7. No hallucinated facts (basic check)
self.register_property(Property(
name="no_obvious_hallucination",
description="Response should not contain obviously false statements",
property_type=PropertyType.INVARIANT,
checker=lambda inp, out: not self._has_obvious_hallucination(str(out))
))
def register_property(self, property_def: Property):
"""Register a new property to test"""
self.properties[property_def.name] = property_def
print(f"ā
Registered property: {property_def.name}")
def test_property(self, property_name: str, test_count: int = 20,
custom_test_cases: Optional[List[Any]] = None) -> PropertyTestReport:
"""
Test a specific property with generated or provided test cases
Args:
property_name: Name of property to test
test_count: Number of test cases to generate
custom_test_cases: Optional list of specific test cases
Returns:
Comprehensive test report
"""
if property_name not in self.properties:
raise ValueError(f"Property '{property_name}' not registered")
property_def = self.properties[property_name]
start_time = time.time()
print(f"\n{'='*70}")
print(f"Testing Property: {property_name}")
print(f"Type: {property_def.property_type.value}")
print(f"Description: {property_def.description}")
print(f"{'='*70}")
# Generate or use provided test cases
if custom_test_cases:
test_cases = custom_test_cases
else:
test_cases = self._generate_test_cases(property_def, test_count)
print(f"\nš Running {len(test_cases)} test cases...")
results = []
for i, test_case in enumerate(test_cases, 1):
result = self._run_test_case(property_def, test_case, i)
results.append(result)
# Analyze results
passed = sum(1 for r in results if r.passed)
failed = len(results) - passed
counterexamples = []
for r in results:
if not r.passed:
counterexamples.append({
"input": r.test_case,
"output": r.output,
"error": r.error
})
execution_time = time.time() - start_time
report = PropertyTestReport(
property_name=property_name,
total_tests=len(results),
passed=passed,
failed=failed,
counterexamples=counterexamples,
execution_time=execution_time
)
self._print_test_results(report)
return report
def test_all_properties(self, test_count: int = 10) -> Dict[str, PropertyTestReport]:
"""Test all registered properties"""
print(f"\n{'='*70}")
print("TESTING ALL PROPERTIES")
print(f"{'='*70}")
print(f"Properties to test: {len(self.properties)}")
print(f"Test cases per property: {test_count}")
reports = {}
for property_name in self.properties:
report = self.test_property(property_name, test_count)
reports[property_name] = report
# Overall summary
self._print_overall_summary(reports)
return reports
def _generate_test_cases(self, property_def: Property, count: int) -> List[str]:
"""Generate test cases for a property"""
print(f" š Generating test cases...")
# Use examples as seed
constraints = "Include diverse scenarios, edge cases, and boundary conditions"
try:
chain = self.test_gen_prompt | self.llm | StrOutputParser()
result = chain.invoke({
"property_description": property_def.description,
"constraints": constraints,
"count": count
})
# Parse test cases (one per line or numbered)
test_cases = []
lines = result.strip().split('\n')
for line in lines:
# Remove numbering/bullets
cleaned = re.sub(r'^\d+[\.\)]\s*', '', line.strip())
cleaned = re.sub(r'^[-ā¢]\s*', '', cleaned)
if cleaned and len(cleaned) > 3:
test_cases.append(cleaned)
# Fill up to count if needed
while len(test_cases) TestResult:
"""Run a single test case"""
start_time = time.time()
try:
# Execute agent with test case
output = self._execute_agent(test_case)
# Check property
if property_def.property_type == PropertyType.CONSISTENCY:
passed = self._check_consistency(test_case)
else:
passed = property_def.check(test_case, output)
result = TestResult(
property_name=property_def.name,
passed=passed,
test_case=test_case,
output=output,
execution_time=time.time() - start_time
)
status = "ā" if passed else "ā"
print(f" {status} Test {case_number}: {str(test_case)[:60]}...")
return result
except Exception as e:
print(f" ā Test {case_number}: ERROR - {str(e)}")
return TestResult(
property_name=property_def.name,
passed=False,
test_case=test_case,
output=None,
error=str(e),
execution_time=time.time() - start_time
)
def _execute_agent(self, query: str) -> str:
"""Execute the agent being tested"""
chain = self.agent_prompt | self.llm | StrOutputParser()
return chain.invoke({"query": query})
def _check_consistency(self, test_case: Any) -> bool:
"""Check consistency property by running multiple times"""
outputs = []
# Run 3 times with temperature 0 for consistency
chain = self.agent_prompt | self.consistent_llm | StrOutputParser()
for _ in range(3):
output = chain.invoke({"query": test_case})
outputs.append(output)
# Check if all outputs are similar (basic check)
# For production, use more sophisticated similarity metrics
return len(set(outputs)) bool:
"""Check if text contains offensive content (basic)"""
offensive_words = ["offensive", "inappropriate", "harmful"] # Simplified
return any(word in text.lower() for word in offensive_words)
def _is_valid_json_if_requested(self, input_text: str, output_text: str) -> bool:
"""Check if output is valid JSON when requested"""
if "json" not in input_text.lower():
return True # Not requesting JSON, so property holds
try:
import json
json.loads(output_text)
return True
except:
# Check if JSON is embedded in response
if "{" in output_text and "}" in output_text:
start = output_text.find("{")
end = output_text.rfind("}") + 1
try:
json.loads(output_text[start:end])
return True
except:
return False
return False
def _has_obvious_hallucination(self, text: str) -> bool:
"""Check for obvious hallucinations (basic)"""
# Check for contradictions or obviously false statements
false_patterns = [
r"the sun is cold",
r"water is dry",
r"earth is flat",
r"humans have \d+ legs" # where number != 2
]
text_lower = text.lower()
for pattern in false_patterns:
if re.search(pattern, text_lower):
return True
return False
def _print_test_results(self, report: PropertyTestReport):
"""Print test results"""
print(f"\n{'='*70}")
print(f"RESULTS: {report.property_name}")
print(f"{'='*70}")
print(f"Total Tests: {report.total_tests}")
print(f"Passed: {report.passed} ({report.pass_rate*100:.1f}%)")
print(f"Failed: {report.failed}")
print(f"Execution Time: {report.execution_time:.2f}s")
if report.counterexamples:
print(f"\nā Counterexamples Found ({len(report.counterexamples)}):")
for i, ce in enumerate(report.counterexamples[:5], 1): # Show first 5
print(f"\n{i}. Input: {str(ce['input'])[:80]}...")
if ce['output']:
print(f" Output: {str(ce['output'])[:80]}...")
if ce['error']:
print(f" Error: {ce['error']}")
print("="*70)
def _print_overall_summary(self, reports: Dict[str, PropertyTestReport]):
"""Print overall summary of all tests"""
print(f"\n{'='*70}")
print("OVERALL TEST SUMMARY")
print(f"{'='*70}")
total_tests = sum(r.total_tests for r in reports.values())
total_passed = sum(r.passed for r in reports.values())
overall_pass_rate = total_passed / total_tests if total_tests > 0 else 0
print(f"\nTotal Tests Run: {total_tests}")
print(f"Overall Pass Rate: {overall_pass_rate*100:.1f}%")
print(f"\nProperty Results:")
for name, report in reports.items():
status = "ā" if report.pass_rate >= 0.9 else "ā" if report.pass_rate Any:
"""
Attempt to find minimal counterexample (shrinking)
Args:
property_name: Property that failed
counterexample: Original counterexample
Returns:
Minimal counterexample
"""
property_def = self.properties[property_name]
current = str(counterexample)
print(f"\nš Shrinking counterexample for {property_name}...")
print(f"Original: {current[:100]}...")
# Try progressively smaller versions
for length in range(len(current) // 2, 10, -10):
candidate = current[:length]
try:
output = self._execute_agent(candidate)
if not property_def.check(candidate, output):
current = candidate
print(f" Shrunk to: {len(current)} characters")
except:
continue
print(f"ā Minimal counterexample: {current}")
return current
def demonstrate_property_based_testing():
"""Demonstrate property-based testing"""
print("="*70)
print("Pattern 152: Property-Based Testing")
print("="*70)
tester = PropertyBasedTester()
# Test 1: Non-empty response property
print("\n" + "="*70)
print("TEST 1: Non-Empty Response Property")
print("="*70)
test_cases = [
"What is Python?",
"Hello",
"", # Edge case
"Explain quantum physics",
"?" # Edge case
]
report1 = tester.test_property("non_empty_response",
custom_test_cases=test_cases)
# Test 2: Reasonable length property
print("\n" + "="*70)
print("TEST 2: Reasonable Length Property")
print("="*70)
report2 = tester.test_property("reasonable_length", test_count=8)
# Test 3: JSON format compliance
print("\n" + "="*70)
print("TEST 3: JSON Format Compliance")
print("="*70)
json_test_cases = [
"Return the result as JSON: {name: 'test'}",
"Give me JSON with user info",
"What is 2+2?", # Should pass (not requesting JSON)
"Provide a JSON object with three fields"
]
report3 = tester.test_property("json_format",
custom_test_cases=json_test_cases)
# Test 4: Custom property
print("\n" + "="*70)
print("TEST 4: Custom Property - No Numbers in Text Explanation")
print("="*70)
def no_numbers_checker(inp: Any, out: Any) -> bool:
"""Custom checker: explanations should avoid raw numbers"""
if "explain" in str(inp).lower():
# Allow some numbers but not excessive
return str(out).count("1") + str(out).count("2") < 10
return True
custom_prop = Property(
name="minimal_numbers_in_explanation",
description="Explanations should minimize raw numbers",
property_type=PropertyType.POSTCONDITION,
checker=no_numbers_checker
)
tester.register_property(custom_prop)
explain_cases = [
"Explain how computers work",
"What is machine learning?",
"Tell me about AI"
]
report4 = tester.test_property("minimal_numbers_in_explanation",
custom_test_cases=explain_cases)
# Test 5: Consistency property
print("\n" + "="*70)
print("TEST 5: Consistency Property")
print("="*70)
consistent_cases = [
"What is 2+2?",
"Name the capital of France"
]
report5 = tester.test_property("consistency",
custom_test_cases=consistent_cases)
# Summary
print("\n" + "="*70)
print("SUMMARY")
print("="*70)
print(f"""
Property-Based Testing Pattern provides:
1. Property Definitions:
- Invariants (must always hold)
- Postconditions (output constraints)
- Consistency checks
- Format compliance
- Custom properties
2. Test Generation:
- Automated test case creation
- Edge case discovery
- Boundary condition testing
- Coverage maximization
3. Property Verification:
- Systematic property checking
- Counterexample identification
- Shrinking to minimal cases
- Detailed reporting
4. Benefits:
- Finds bugs traditional testing misses
- Documents expected behaviors
- Scales testing automatically
- More thorough coverage
5. Results Summary:
- Non-empty responses: {report1.pass_rate*100:.0f}% pass
- Reasonable length: {report2.pass_rate*100:.0f}% pass
- JSON format: {report3.pass_rate*100:.0f}% pass
- Custom property: {report4.pass_rate*100:.0f}% pass
- Consistency: {report5.pass_rate*100:.0f}% pass
This pattern is essential for robust AI agent testing and
quality assurance in production systems.
""")
if __name__ == "__main__":
demonstrate_property_based_testing()
Agentic Design Pattern: Property Based Testing
Pattern 152: Property Based Testing – Line-by-Line Explanation
Overview
This document provides a detailed line-by-line explanation of the Property Based Testing pattern implementation.
Pattern Description
Pattern 152: Property-Based Testing
Description: Implements property-based testing for AI agents by defining invariants and properties that should hold across all inputs, then generating test cases to verify these properties.
Components: – Property Definitions: Invariants that should always hold – Test Case Generators: Create diverse inputs – Property Verifiers: Check if properties hold – Hypothesis Engine: Generate test hypotheses – Counterexample Finder: Identify violations
Use Cases: – Testing agent robustness across input space – Finding edge cases and failures – Verifying consistency properties – Ensuring output format compliance – Testing deterministic vs non-deterministic behaviors
Benefits: – Discovers unexpected edge cases – More thorough than example-based testing – Finds bugs traditional testing misses – Documents expected behaviors as properties – Scales testing coverage automatically
Trade-offs: – Requires careful property definition – Can be slower than unit tests – May generate false positives – Harder to debug failures – Non-determinism complicates verification
LangChain Implementation: Uses ChatOpenAI with property checking chains and systematic test case generation.
File Statistics
- Total Lines: 644
- Classes: 5
- Functions: 1
- Constants: 0
Module Documentation
"""
Pattern 152: Property-Based Testing
Description:
Implements property-based testing for AI agents by defining invariants
and properties that should hold across all inputs, then generating test
cases to verify these properties.
Components:
- Property Definitions: Invariants that should always hold
- Test Case Generators: Create diverse inputs
- Property Verifiers: Check if properties hold
- Hypothesis Engine: Generate test hypotheses
- Counterexample Finder: Identify violations
Use Cases:
- Testing agent robustness across input space
- Finding edge cases and failures
- Verifying consistency properties
- Ensuring output format compliance
- Testing deterministic vs non-deterministic behaviors
Benefits:
- Discovers unexpected edge cases
- More thorough than example-based testing
- Finds bugs traditional testing misses
- Documents expected behaviors as properties
- Scales testing coverage automatically
Trade-offs:
- Requires careful property definition
- Can be slower than unit tests
- May generate false positives
- Harder to debug failures
- Non-determinism complicates verification
LangChain Implementation:
Uses ChatOpenAI with property checking chains and systematic
test case generation.
"""
Purpose: This module implements the Property Based Testing pattern, providing a structured approach to agent design and implementation.
Import Statements
Standard Library Imports
import collections.defaultdict
import dataclasses.dataclass
import dataclasses.field
import datetime.datetime
import enum.Enum
import json
import langchain_core.output_parsers.StrOutputParser
import langchain_core.prompts.ChatPromptTemplate
import os
import re
import time
import typing.Any
import typing.Callable
import typing.Dict
import typing.List
import typing.Optional
import typing.Tuple
Purpose: Core Python functionality:
– collections.defaultdict: Specialized container datatypes
– dataclasses.dataclass: Simplified class definitions with @dataclass
– dataclasses.field: Simplified class definitions with @dataclass
– datetime.datetime: Date and time handling
– enum.Enum: Enumeration support
– json: JSON serialization/deserialization
– langchain_core.output_parsers.StrOutputParser: Regular expression operations
– langchain_core.prompts.ChatPromptTemplate: Regular expression operations
– os: Environment variable access and system operations
– re: Regular expression operations
– time: Time-related functions
– typing.Any: Type hints for better code documentation (List, Dict, Any, Optional)
– typing.Callable: Type hints for better code documentation (List, Dict, Any, Optional)
– typing.Dict: Type hints for better code documentation (List, Dict, Any, Optional)
– typing.List: Type hints for better code documentation (List, Dict, Any, Optional)
– typing.Optional: Type hints for better code documentation (List, Dict, Any, Optional)
– typing.Tuple: Type hints for better code documentation (List, Dict, Any, Optional)
LangChain Imports
from langchain_openai.ChatOpenAI
Purpose: LangChain framework components: – LLM Interface: Wrapper for OpenAI models – Chains: Composable building blocks for complex workflows
Other Imports
from dotenv.load_dotenv
Purpose: Additional dependencies for specialized functionality.
Classes
Class: PropertyType
Lines 59-67
Inherits from: Enum
Purpose:
Types of properties to test
Class: Property
Lines 71-81
Purpose:
Defines a testable property
Methods:
check(self, input_val, output_val)
Lines 79-81
Purpose:
Check if property holds
Parameters:
– input_val: Input data
– output_val: Output val
Returns: bool
Class: TestResult
Lines 85-93
Purpose:
Result of a property test
Class: PropertyTestReport
Lines 97-108
Purpose:
Comprehensive test report
Methods:
pass_rate(self)
Lines 107-108
Decorators: property
Purpose: Pass Rate operation.
Parameters:
Returns: float
Class: PropertyBasedTester
Lines 111-506
Purpose:
Agent that performs property-based testing on AI systems.
Generates test cases and verifies properties hold.
Methods:
__init__(self, temperature)
Lines 117-147
Purpose:
Initialize the property-based tester
Parameters:
– temperature: Sampling temperature (0-1)
_register_default_properties(self)
Lines 149-206
Purpose:
Register commonly used properties
Parameters:
register_property(self, property_def)
Lines 208-211
Purpose:
Register a new property to test
Parameters:
– property_def: Property def
test_property(self, property_name, test_count, custom_test_cases)
Lines 213-277
Purpose:
Test a specific property with generated or provided test cases
Args:
property_name: Name of property to test
test_count: Number of test cases to generate
custom_test_cases: Optional list of specific test cases
Returns:
Comprehensive test report
Parameters:
– property_name: Property name
– test_count: Test count
– custom_test_cases: Custom test cases
Returns: PropertyTestReport
test_all_properties(self, test_count)
Lines 279-295
Purpose:
Test all registered properties
Parameters:
– test_count: Test count
Returns: <ast.Subscript object at 0x000001A06D617AD0>
_generate_test_cases(self, property_def, count)
Lines 297-331
Purpose:
Generate test cases for a property
Parameters:
– property_def: Property def
– count: Count
Returns: <ast.Subscript object at 0x000001A06D7CFC90>
_run_test_case(self, property_def, test_case, case_number)
Lines 333-370
Purpose:
Run a single test case
Parameters:
– property_def: Property def
– test_case: Test case
– case_number: Case number
Returns: TestResult
_execute_agent(self, query)
Lines 372-375
Purpose:
Execute the agent being tested
Parameters:
– query: Input query or question
Returns: str
_check_consistency(self, test_case)
Lines 377-389
Purpose:
Check consistency property by running multiple times
Parameters:
– test_case: Test case
Returns: bool
_contains_offensive(self, text)
Lines 391-394
Purpose:
Check if text contains offensive content (basic)
Parameters:
– text: Text
Returns: bool
_is_valid_json_if_requested(self, input_text, output_text)
Lines 396-415
Purpose:
Check if output is valid JSON when requested
Parameters:
– input_text: Input data
– output_text: Output text
Returns: bool
_has_obvious_hallucination(self, text)
Lines 417-432
Purpose:
Check for obvious hallucinations (basic)
Parameters:
– text: Text
Returns: bool
_print_test_results(self, report)
Lines 434-453
Purpose:
Print test results
Parameters:
– report: Report
_print_overall_summary(self, reports)
Lines 455-474
Purpose:
Print overall summary of all tests
Parameters:
– reports: Reports
shrink_counterexample(self, property_name, counterexample)
Lines 476-506
Purpose:
Attempt to find minimal counterexample (shrinking)
Args:
property_name: Property that failed
counterexample: Original counterexample
Returns:
Minimal counterexample
Parameters:
– property_name: Property name
– counterexample: Counterexample
Returns: Any
Functions
demonstrate_property_based_testing()
Lines 509-639
Purpose:
Demonstrate property-based testing
Key Concepts Summary
Pattern Overview
The Property Based Testing pattern provides a structured approach to building AI agents with specific capabilities and behaviors. This implementation leverages LangChain’s composable components to create a robust and maintainable solution.
Core Components
- Agent Class: Main implementation of the pattern logic
- Prompt Engineering: Carefully crafted prompts to guide LLM behavior
- Tool Integration: Optional tools for extended functionality
- Memory Management: State handling across interactions
- Output Parsing: Structured response processing
When to Use
ā Good for: – Scenarios requiring this specific pattern’s strengths – Applications needing structured agent behavior – Systems where explainability is important – Production environments requiring reliability
ā Not ideal for: – Simple single-prompt tasks – Scenarios requiring different pattern characteristics – Resource-constrained environments (if computationally intensive)
Best Practices Demonstrated
1. Environment Configuration
- ā
Use
.envfiles for API keys - ā Never hardcode credentials
- ā Validate environment variables on startup
2. Error Handling
- ā Graceful degradation on failures
- ā Informative error messages
- ā Logging for debugging
3. Prompt Engineering
- ā Clear, specific instructions
- ā Structured output formats
- ā Few-shot examples when beneficial
4. Code Organization
- ā Separation of concerns (class-based design)
- ā Type hints for clarity
- ā Docstrings for documentation
5. Performance Optimization
- ā Appropriate temperature settings
- ā Token budget management
- ā Caching where applicable
LangChain Components Used
- ChatOpenAI: OpenAI model interface
- Chains: Sequential component composition
Why LangChain? – Composable abstractions for LLM applications – Rich ecosystem of integrations – Production-ready patterns – Active community and development
Execution Flow
1. Initialize agent with configuration
ā
2. Load environment and dependencies
ā
3. Prepare prompt with input data
ā
4. Invoke LLM through LangChain
ā
5. Parse and validate response
ā
6. Return formatted result
Key Decision Points: – Model selection (temperature, model type) – Prompt structure (few-shot vs zero-shot) – Error handling strategy – Output format validation
Performance Considerations
Computational Cost
- LLM Calls: Primary cost factor (based on tokens)
- Latency: Depends on model and input size
- Memory: Minimal for most patterns
Optimization Strategies
- Prompt Optimization: Minimize tokens while maintaining clarity
- Caching: Cache repeated queries/responses
- Model Selection: Use smaller models when appropriate
- Batch Processing: Group similar requests
- Streaming: For long responses, stream tokens
Cost Estimates (Approximate)
- GPT-3.5-turbo: ~$0.002 per request
- GPT-4: ~$0.03 per request
- GPT-4-turbo: ~$0.01 per request
Actual costs vary based on input/output token counts
Common Pitfalls to Avoid
1. Prompt Design Issues
ā Vague or ambiguous instructions
ā
Clear, specific, testable prompts
2. Error Handling
ā Ignoring API errors or rate limits
ā
Implement retry logic and graceful degradation
3. Token Management
ā Exceeding model token limits
ā
Truncate inputs, chunk processing
4. Security
ā Exposing API keys in code
ā
Use environment variables
5. Testing
ā Only testing happy paths
ā
Test edge cases, failures, malformed inputs
6. Cost Management
ā Unmonitored API usage
ā
Implement budgets, alerts, and monitoring
Further Improvements
Production Enhancements
- Monitoring: Add logging, metrics, tracing
- Validation: Input/output schema validation
- Rate Limiting: Implement request throttling
- Caching: Redis or in-memory cache for responses
- Async Support: Non-blocking operations for scalability
Feature Extensions
- Multi-Model Support: Switch between providers
- Fallback Models: Graceful degradation strategy
- Custom Tools: Domain-specific function calling
- Advanced Memory: Vector stores for long-term memory
- Evaluation: Automated testing and benchmarking
Code Quality
- Testing: Unit tests, integration tests
- Documentation: API docs, usage examples
- Type Safety: Stricter type hints
- Configuration: Externalized config files
Related Patterns
- Pattern 151
- Pattern 153
Pattern Families: – Reasoning Patterns (001-011) – Multi-Agent Patterns (012-019) – Tool Use Patterns (020-025) – Memory Patterns (026-031) – And many more…
Summary
The Property Based Testing pattern provides a powerful approach to building AI agents with specific capabilities. This implementation demonstrates:
ā Key Strengths: – Clean, maintainable code structure – LangChain integration for composability – Type hints and documentation – Production-ready patterns
šÆ Best Use Cases: – Applications requiring this pattern’s characteristics – Systems needing structured agent behavior – Production environments requiring reliability
š Learning Resources: – LangChain Documentation: https://python.langchain.com/ – Pattern Documentation: See related patterns – OpenAI API Reference: https://platform.openai.com/docs/
File: 152_property_based_testing.py
Total Lines: 644
Pattern Category: AI Agent Design Pattern #152
Generated from actual implementation code
For educational and reference purposes
Leave a Reply