# Example where traditional metrics struggle
question = "I'm really frustrated. I've been trying to cancel my subscription for weeks and keep getting the runaround. Can someone please help me?"

response_a = "To cancel your subscription, log into your account and click the cancel button in settings."
response_b = "I understand how frustrating this must be for you. Let me personally help you cancel your subscription right now. I'll also make sure you get a refund for any days you couldn't use the service due to this issue."

# Traditional metrics might favor Response A (shorter, more direct)
# But humans clearly prefer Response B (empathetic, comprehensive, proactive)

import openai
import json
from typing import Dict, List

class LLMJudge:
    """
    A sophisticated LLM-based evaluation system
    """

    def __init__(self, model="gpt-4"):
        self.model = model
        self.evaluation_criteria = {
            "accuracy": "Is the information factually correct?",
            "completeness": "Does the response fully address the question?",
            "clarity": "Is the response clear and easy to understand?",
            "helpfulness": "Would this response actually help the user?",
            "tone": "Is the tone appropriate for the context?",
            "safety": "Is the response safe and free from harmful content?"
        }

    def create_evaluation_prompt(self, question: str, response: str, criteria: List[str] = None) -> str:
        """
        Create a detailed prompt for LLM evaluation
        """
        if criteria is None:
            criteria = list(self.evaluation_criteria.keys())

        criteria_descriptions = "\n".join([
            f"- {criterion}: {self.evaluation_criteria[criterion]}"
            for criterion in criteria
        ])

        prompt = f"""
You are an expert evaluator tasked with assessing the quality of an AI assistant's response. Please evaluate the response carefully and objectively.

QUESTION FROM USER:
{question}

RESPONSE TO EVALUATE:
{response}

EVALUATION CRITERIA:
{criteria_descriptions}

INSTRUCTIONS:
1. For each criterion, provide a score from 1-5 where:
   - 1 = Poor (major issues, significant problems)
   - 2 = Below Average (notable problems, some issues)
   - 3 = Average (adequate, meets basic requirements)
   - 4 = Good (solid quality, minor room for improvement)
   - 5 = Excellent (exceptional quality, exceeds expectations)

2. For each score, provide a brief explanation (1-2 sentences) justifying your rating.

3. Provide an overall assessment and suggestions for improvement.

Please respond in the following JSON format:
{{
    "scores": {{
        "accuracy": {{"score": X, "explanation": "..."}},
        "completeness": {{"score": X, "explanation": "..."}},
        "clarity": {{"score": X, "explanation": "..."}},
        "helpfulness": {{"score": X, "explanation": "..."}},
        "tone": {{"score": X, "explanation": "..."}},
        "safety": {{"score": X, "explanation": "..."}}
    }},
    "overall_score": X.X,
    "overall_assessment": "...",
    "improvement_suggestions": ["...", "...", "..."]
}}
"""
        return prompt

    def evaluate_response(self, question: str, response: str, criteria: List[str] = None) -> Dict:
        """
        Evaluate a response using the LLM judge
        """
        prompt = self.create_evaluation_prompt(question, response, criteria)

        try:
            # Call the LLM (using OpenAI as example)
            response = openai.ChatCompletion.create(
                model=self.model,
                messages=[
                    {"role": "system", "content": "You are a precise and fair evaluator."},
                    {"role": "user", "content": prompt}
                ],
                temperature=0.1  # Low temperature for consistent evaluation
            )

            # Parse the JSON response
            evaluation = json.loads(response.choices[0].message.content)
            return evaluation

        except Exception as e:
            print(f"Error in LLM evaluation: {e}")
            return {"error": str(e)}

    def batch_evaluate(self, test_cases: List[Dict]) -> List[Dict]:
        """
        Evaluate multiple test cases efficiently
        """
        results = []

        print(f"🤖 Evaluating {len(test_cases)} responses with LLM Judge...")
        print("=" * 60)

        for i, test_case in enumerate(test_cases, 1):
            print(f"\n📝 Evaluating Response {i}/{len(test_cases)}")

            evaluation = self.evaluate_response(
                test_case['question'],
                test_case['response'],
                test_case.get('criteria', None)
            )

            # Add metadata
            evaluation['question'] = test_case['question']
            evaluation['response'] = test_case['response']
            evaluation['test_case_id'] = test_case.get('id', f'case_{i}')

            results.append(evaluation)

            # Print summary
            if 'overall_score' in evaluation:
                print(f"   🏆 Overall Score: {evaluation['overall_score']:.1f}/5.0")
                print(f"   📊 Key Strengths: {self._extract_strengths(evaluation)}")
                if evaluation.get('improvement_suggestions'):
                    print(f"   🎯 Top Improvement: {evaluation['improvement_suggestions'][0]}")

        return results

    def _extract_strengths(self, evaluation: Dict) -> str:
        """Extract the highest-scoring criteria as strengths"""
        if 'scores' not in evaluation:
            return "N/A"

        strengths = []
        for criterion, details in evaluation['scores'].items():
            if details['score'] >= 4:
                strengths.append(criterion)

        return ", ".join(strengths) if strengths else "Areas for improvement identified"

# Example usage
judge = LLMJudge()

test_cases = [
    {
        "id": "customer_service_1",
        "question": "I'm having trouble logging into my account. I've tried resetting my password but never received the email.",
        "response": "I understand how frustrating login issues can be. Let me help you troubleshoot this step by step. First, please check your spam/junk folder as reset emails sometimes end up there. If it's not there, I can manually send you a new reset link or help you update your email address if needed. Would you like me to start with sending a new reset email?"
    },
    {
        "id": "technical_support_1", 
        "question": "Why is my software running so slowly?",
        "response": "Slow software can be caused by many factors. Try restarting your computer."
    }
]

evaluation_results = judge.batch_evaluate(test_cases)

# Analyze results
def analyze_evaluation_results(results: List[Dict]):
    """
    Analyze patterns in LLM judge evaluations
    """
    print("\n📊 EVALUATION ANALYSIS")
    print("=" * 40)

    # Calculate average scores
    all_scores = []
    criterion_scores = {}

    for result in results:
        if 'overall_score' in result:
            all_scores.append(result['overall_score'])

            for criterion, details in result['scores'].items():
                if criterion not in criterion_scores:
                    criterion_scores[criterion] = []
                criterion_scores[criterion].append(details['score'])

    # Overall statistics
    if all_scores:
        avg_score = sum(all_scores) / len(all_scores)
        print(f"📈 Average Overall Score: {avg_score:.2f}/5.0")

        print(f"\n📊 Criterion Breakdown:")
        for criterion, scores in criterion_scores.items():
            avg_criterion = sum(scores) / len(scores)
            print(f"   {criterion.capitalize()}: {avg_criterion:.2f}/5.0")

    # Identify patterns
    print(f"\n🔍 Insights:")
    strengths = []
    weaknesses = []

    for criterion, scores in criterion_scores.items():
        avg_score = sum(scores) / len(scores)
        if avg_score >= 4.0:
            strengths.append(criterion)
        elif avg_score <= 2.5:
            weaknesses.append(criterion)

    if strengths:
        print(f"   💪 System Strengths: {', '.join(strengths)}")
    if weaknesses:
        print(f"   🎯 Areas for Improvement: {', '.join(weaknesses)}")

analyze_evaluation_results(evaluation_results)

class MultiJudgeSystem:
    """
    System using multiple LLM judges for consensus evaluation
    """

    def __init__(self, models=["gpt-4", "gpt-3.5-turbo", "claude-3"]):
        self.judges = [LLMJudge(model) for model in models]
        self.models = models

    def consensus_evaluate(self, question: str, response: str) -> Dict:
        """
        Get evaluations from multiple judges and compute consensus
        """
        individual_evaluations = []

        print(f"🎭 Getting evaluations from {len(self.judges)} judges...")

        for i, judge in enumerate(self.judges):
            print(f"   Judge {i+1} ({self.models[i]}) evaluating...")
            evaluation = judge.evaluate_response(question, response)
            individual_evaluations.append(evaluation)

        # Calculate consensus scores
        consensus = self._calculate_consensus(individual_evaluations)
        return consensus

    def _calculate_consensus(self, evaluations: List[Dict]) -> Dict:
        """
        Calculate consensus from multiple evaluations
        """
        consensus_scores = {}
        overall_scores = []

        # Aggregate scores for each criterion
        for eval_result in evaluations:
            if 'scores' in eval_result:
                for criterion, details in eval_result['scores'].items():
                    if criterion not in consensus_scores:
                        consensus_scores[criterion] = []
                    consensus_scores[criterion].append(details['score'])

                if 'overall_score' in eval_result:
                    overall_scores.append(eval_result['overall_score'])

        # Calculate averages and agreement
        final_consensus = {
            "consensus_scores": {},
            "individual_evaluations": evaluations,
            "agreement_metrics": {}
        }

        for criterion, scores in consensus_scores.items():
            avg_score = sum(scores) / len(scores)
            score_variance = np.var(scores)

            final_consensus["consensus_scores"][criterion] = {
                "average_score": avg_score,
                "individual_scores": scores,
                "agreement_level": "high" if score_variance < 0.5 else "medium" if score_variance < 1.0 else "low"
            }

        if overall_scores:
            final_consensus["consensus_overall_score"] = sum(overall_scores) / len(overall_scores)
            final_consensus["overall_agreement"] = "high" if np.var(overall_scores) < 0.25 else "medium"

        return final_consensus

# Example usage
multi_judge = MultiJudgeSystem()
consensus_result = multi_judge.consensus_evaluate(
    "How do I troubleshoot network connectivity issues?",
    "Check your internet connection and restart your router. If that doesn't work, contact your ISP."
)

class SpecializedLLMJudge(LLMJudge):
    """
    LLM Judge with specialized prompts for different domains
    """

    def __init__(self, domain="general"):
        super().__init__()
        self.domain = domain
        self.domain_prompts = {
            "customer_service": self._customer_service_prompt,
            "technical_support": self._technical_support_prompt,
            "medical": self._medical_prompt,
            "legal": self._legal_prompt,
            "educational": self._educational_prompt
        }

    def _customer_service_prompt(self, question: str, response: str) -> str:
        return f"""
You are evaluating a customer service response. Focus on these key aspects:

CUSTOMER QUESTION: {question}
RESPONSE TO EVALUATE: {response}

EVALUATION FOCUS:
- Empathy: Does the response acknowledge the customer's feelings?
- Problem-solving: Does it provide actionable solutions?
- Professionalism: Is the tone appropriate and respectful?
- Completeness: Are all customer concerns addressed?
- Proactivity: Does it anticipate follow-up needs?

Rate each aspect 1-5 and provide specific feedback for customer service improvement.
"""

    def _technical_support_prompt(self, question: str, response: str) -> str:
        return f"""
You are evaluating a technical support response. Focus on these technical aspects:

TECHNICAL QUESTION: {question}
RESPONSE TO EVALUATE: {response}

EVALUATION FOCUS:
- Technical accuracy: Is the information technically correct?
- Clarity: Are technical concepts explained clearly?
- Step-by-step guidance: Are instructions easy to follow?
- Troubleshooting approach: Does it follow logical diagnostic steps?
- Safety considerations: Are there any safety warnings where needed?

Rate each aspect 1-5 with specific technical feedback.
"""

    def evaluate_domain_specific(self, question: str, response: str) -> Dict:
        """
        Evaluate using domain-specific criteria
        """
        if self.domain in self.domain_prompts:
            prompt = self.domain_prompts[self.domain](question, response)
        else:
            prompt = self.create_evaluation_prompt(question, response)

        # Add domain-specific processing here
        return self._process_domain_evaluation(prompt)

# Example usage for different domains
cs_judge = SpecializedLLMJudge(domain="customer_service")
tech_judge = SpecializedLLMJudge(domain="technical_support")

from sentence_transformers import SentenceTransformer, util
import numpy as np

class AdvancedSemanticEvaluator:
    """
    Advanced semantic evaluation using modern embeddings
    """

    def __init__(self, model_name="all-MiniLM-L6-v2"):
        self.model = SentenceTransformer(model_name)
        print(f"🧠 Loaded embedding model: {model_name}")

    def evaluate_semantic_similarity(self, generated_answers: List[str], 
                                   reference_answers: List[str]) -> Dict:
        """
        Evaluate semantic similarity using state-of-the-art embeddings
        """
        print("🔬 Advanced Semantic Similarity Evaluation")
        print("=" * 50)

        # Generate embeddings
        generated_embeddings = self.model.encode(generated_answers)
        reference_embeddings = self.model.encode(reference_answers)

        similarities = []
        detailed_results = []

        for i, (gen_text, ref_text) in enumerate(zip(generated_answers, reference_answers)):
            # Calculate cosine similarity
            similarity = util.cos_sim(generated_embeddings[i], reference_embeddings[i]).item()
            similarities.append(similarity)

            # Detailed analysis
            result = {
                "pair_id": i + 1,
                "generated_text": gen_text,
                "reference_text": ref_text,
                "semantic_similarity": similarity,
                "quality_level": self._interpret_semantic_score(similarity),
                "embedding_analysis": self._analyze_embeddings(
                    generated_embeddings[i], 
                    reference_embeddings[i]
                )
            }
            detailed_results.append(result)

            print(f"\n📝 Pair {i+1}:")
            print(f"   Generated: '{gen_text[:60]}...'")
            print(f"   Reference: '{ref_text[:60]}...'")
            print(f"   🎯 Similarity: {similarity:.3f} ({result['quality_level']})")

        average_similarity = np.mean(similarities)

        return {
            "individual_similarities": similarities,
            "average_similarity": average_similarity,
            "detailed_results": detailed_results,
            "overall_assessment": self._interpret_semantic_score(average_similarity)
        }

    def _interpret_semantic_score(self, score: float) -> str:
        """Interpret semantic similarity scores"""
        if score >= 0.85:
            return "Excellent semantic match"
        elif score >= 0.70:
            return "Good semantic alignment"
        elif score >= 0.55:
            return "Moderate semantic similarity"
        elif score >= 0.35:
            return "Low semantic similarity"
        else:
            return "Poor semantic match"

    def _analyze_embeddings(self, emb1: np.ndarray, emb2: np.ndarray) -> Dict:
        """
        Analyze embedding characteristics
        """
        # Calculate various distance metrics
        cosine_sim = util.cos_sim(emb1, emb2).item()
        euclidean_dist = np.linalg.norm(emb1 - emb2)
        manhattan_dist = np.sum(np.abs(emb1 - emb2))

        return {
            "cosine_similarity": cosine_sim,
            "euclidean_distance": euclidean_dist,
            "manhattan_distance": manhattan_dist,
            "embedding_magnitude_diff": abs(np.linalg.norm(emb1) - np.linalg.norm(emb2))
        }

# Example usage
semantic_evaluator = AdvancedSemanticEvaluator()

test_generated = [
    "To reset your password, go to settings and click forgot password.",
    "We're open Monday through Friday from 9 AM to 5 PM.",
    "You can track your order in the my orders section of your account."
]

test_reference = [
    "Password reset: Navigate to account settings, select 'forgot password', and follow the email instructions.",
    "Our business hours are Monday-Friday 9:00 AM to 5:00 PM.",
    "Track your package by logging into your account and checking the order status page."
]

semantic_results = semantic_evaluator.evaluate_semantic_similarity(test_generated, test_reference)

class ContextAwareEvaluator:
    """
    Evaluator that considers conversation context and user intent
    """

    def __init__(self):
        self.semantic_model = SentenceTransformer('all-MiniLM-L6-v2')

    def evaluate_with_context(self, conversation_history: List[Dict], 
                            current_response: str, expected_response: str) -> Dict:
        """
        Evaluate response considering full conversation context
        """
        print("🔄 Context-Aware Evaluation")
        print("=" * 40)

        # Extract context features
        context_analysis = self._analyze_context(conversation_history)

        # Evaluate response appropriateness for context
        context_score = self._score_context_appropriateness(
            conversation_history, current_response
        )

        # Evaluate response quality
        quality_score = self._score_response_quality(
            current_response, expected_response
        )

        # Evaluate conversation flow
        flow_score = self._score_conversation_flow(
            conversation_history, current_response
        )

        # Combine scores
        overall_score = (
            0.4 * quality_score +
            0.3 * context_score +
            0.3 * flow_score
        )

        result = {
            "context_analysis": context_analysis,
            "context_appropriateness": context_score,
            "response_quality": quality_score,
            "conversation_flow": flow_score,
            "overall_score": overall_score,
            "detailed_feedback": self._generate_context_feedback(
                context_analysis, context_score, quality_score, flow_score
            )
        }

        print(f"📊 Context Appropriateness: {context_score:.3f}")
        print(f"📊 Response Quality: {quality_score:.3f}")
        print(f"📊 Conversation Flow: {flow_score:.3f}")
        print(f"🏆 Overall Score: {overall_score:.3f}")

        return result

    def _analyze_context(self, history: List[Dict]) -> Dict:
        """Analyze conversation context characteristics"""
        if not history:
            return {"context_type": "initial", "complexity": "low"}

        # Analyze conversation characteristics
        turn_count = len(history)
        total_length = sum(len(turn.get('text', '')) for turn in history)
        avg_turn_length = total_length / turn_count if turn_count > 0 else 0

        # Identify conversation patterns
        question_count = sum(1 for turn in history if '?' in turn.get('text', ''))
        problem_indicators = ['problem', 'issue', 'error', 'trouble', 'help']
        problem_mentions = sum(
            1 for turn in history 
            for indicator in problem_indicators 
            if indicator.lower() in turn.get('text', '').lower()
        )

        return {
            "turn_count": turn_count,
            "avg_turn_length": avg_turn_length,
            "question_ratio": question_count / turn_count if turn_count > 0 else 0,
            "problem_complexity": "high" if problem_mentions > 2 else "medium" if problem_mentions > 0 else "low",
            "conversation_type": self._classify_conversation_type(history)
        }

    def _classify_conversation_type(self, history: List[Dict]) -> str:
        """Classify the type of conversation"""
        if not history:
            return "initial_inquiry"

        recent_text = " ".join([turn.get('text', '') for turn in history[-3:]]).lower()

        if any(word in recent_text for word in ['thank', 'thanks', 'solved', 'fixed']):
            return "resolution"
        elif any(word in recent_text for word in ['still', 'not working', 'didn\'t work']):
            return "escalation"
        elif any(word in recent_text for word in ['how', 'what', 'when', 'where']):
            return "information_seeking"
        else:
            return "general_inquiry"

# Example usage
context_evaluator = ContextAwareEvaluator()

conversation_example = [
    {"role": "user", "text": "I'm having trouble logging into my account"},
    {"role": "assistant", "text": "I'd be happy to help you with login issues. Can you tell me what happens when you try to log in?"},
    {"role": "user", "text": "It says my password is incorrect, but I'm sure it's right"},
    {"role": "assistant", "text": "That's frustrating! Let's try resetting your password. I'll send you a reset link."},
    {"role": "user", "text": "I tried that already but never got the email"}
]

current_response = "Let me check your email address and send the reset link to a different email if needed. Can you verify the email address on your account?"
expected_response = "I'll help you troubleshoot the email issue. First, let's check if the email went to your spam folder, then I can manually send a new reset link."

context_result = context_evaluator.evaluate_with_context(
    conversation_example, current_response, expected_response
)

import logging
import asyncio
from datetime import datetime
from typing import Optional
import pandas as pd

class ProductionEvaluationPipeline:
    """
    Production-ready evaluation pipeline with monitoring and scaling
    """

    def __init__(self, config: Dict):
        self.config = config
        self.setup_logging()
        self.evaluation_history = []

        # Initialize evaluators
        self.semantic_evaluator = AdvancedSemanticEvaluator()
        self.llm_judge = LLMJudge()
        self.context_evaluator = ContextAwareEvaluator()

    def setup_logging(self):
        """Setup comprehensive logging for evaluation pipeline"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('evaluation_pipeline.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)

    async def run_comprehensive_evaluation(self, test_dataset: List[Dict]) -> Dict:
        """
        Run complete evaluation pipeline with multiple methods
        """
        self.logger.info(f"🚀 Starting comprehensive evaluation of {len(test_dataset)} test cases")

        start_time = datetime.now()

        try:
            # Run evaluations in parallel where possible
            tasks = []

            # Traditional metrics
            tasks.append(self._run_traditional_metrics(test_dataset))

            # Semantic evaluation
            tasks.append(self._run_semantic_evaluation(test_dataset))

            # LLM judge evaluation (run in batches to manage API limits)
            tasks.append(self._run_llm_judge_evaluation(test_dataset))

            # Context-aware evaluation
            tasks.append(self._run_context_evaluation(test_dataset))

            # Execute all evaluations
            results = await asyncio.gather(*tasks, return_exceptions=True)

            # Combine results
            combined_results = self._combine_evaluation_results(results)

            # Calculate execution time
            execution_time = (datetime.now() - start_time).total_seconds()

            # Generate comprehensive report
            evaluation_report = self._generate_evaluation_report(
                combined_results, execution_time, len(test_dataset)
            )

            # Store results
            self._store_evaluation_results(evaluation_report)

            self.logger.info(f"✅ Evaluation completed in {execution_time:.2f} seconds")

            return evaluation_report

        except Exception as e:
            self.logger.error(f"❌ Evaluation pipeline failed: {str(e)}")
            raise

    async def _run_traditional_metrics(self, dataset: List[Dict]) -> Dict:
        """Run traditional evaluation metrics"""
        self.logger.info("📊 Running traditional metrics evaluation")

        # Extract data for traditional metrics
        generated_answers = [item['generated_response'] for item in dataset]
        reference_answers = [item['reference_response'] for item in dataset]

        # Calculate ROUGE scores
        rouge_results = comprehensive_rouge_evaluation(generated_answers, reference_answers)

        return {
            "method": "traditional_metrics",
            "rouge_scores": rouge_results,
            "execution_time": 0.5  # Placeholder
        }

    async def _run_semantic_evaluation(self, dataset: List[Dict]) -> Dict:
        """Run semantic similarity evaluation"""
        self.logger.info("🧠 Running semantic evaluation")

        generated_answers = [item['generated_response'] for item in dataset]
        reference_answers = [item['reference_response'] for item in dataset]

        semantic_results = self.semantic_evaluator.evaluate_semantic_similarity(
            generated_answers, reference_answers
        )

        return {
            "method": "semantic_evaluation",
            "results": semantic_results,
            "execution_time": 2.1  # Placeholder
        }

    async def _run_llm_judge_evaluation(self, dataset: List[Dict]) -> Dict:
        """Run LLM judge evaluation with rate limiting"""
        self.logger.info("🤖 Running LLM judge evaluation")

        # Process in batches to respect API limits
        batch_size = self.config.get('llm_batch_size', 5)
        all_results = []

        for i in range(0, len(dataset), batch_size):
            batch = dataset[i:i + batch_size]
            batch_test_cases = [
                {
                    'question': item['question'],
                    'response': item['generated_response'],
                    'id': item.get('id', f'test_{i+j}')
                }
                for j, item in enumerate(batch)
            ]

            batch_results = self.llm_judge.batch_evaluate(batch_test_cases)
            all_results.extend(batch_results)

            # Rate limiting delay
            await asyncio.sleep(1)

        return {
            "method": "llm_judge",
            "results": all_results,
            "execution_time": 15.7  # Placeholder
        }

    def _generate_evaluation_report(self, combined_results: Dict, 
                                  execution_time: float, dataset_size: int) -> Dict:
        """Generate comprehensive evaluation report"""

        report = {
            "evaluation_metadata": {
                "timestamp": datetime.now().isoformat(),
                "dataset_size": dataset_size,
                "execution_time_seconds": execution_time,
                "evaluation_methods": list(combined_results.keys())
            },
            "summary_metrics": {},
            "detailed_results": combined_results,
            "insights_and_recommendations": [],
            "quality_gates": self._check_quality_gates(combined_results)
        }

        # Calculate summary metrics
        if "semantic_evaluation" in combined_results:
            semantic_avg = combined_results["semantic_evaluation"]["results"]["average_similarity"]
            report["summary_metrics"]["average_semantic_similarity"] = semantic_avg

        if "llm_judge" in combined_results:
            llm_scores = [r.get("overall_score", 0) for r in combined_results["llm_judge"]["results"]]
            if llm_scores:
                report["summary_metrics"]["average_llm_judge_score"] = sum(llm_scores) / len(llm_scores)

        # Generate insights
        report["insights_and_recommendations"] = self._generate_insights(combined_results)

        return report

    def _check_quality_gates(self, results: Dict) -> Dict:
        """
        Check if evaluation results meet quality thresholds
        """
        gates = {
            "semantic_similarity_threshold": {
                "threshold": 0.7,
                "passed": False,
                "actual_value": None
            },
            "llm_judge_threshold": {
                "threshold": 3.5,
                "passed": False,
                "actual_value": None
            }
        }

        # Check semantic similarity gate
        if "semantic_evaluation" in results:
            avg_similarity = results["semantic_evaluation"]["results"]["average_similarity"]
            gates["semantic_similarity_threshold"]["actual_value"] = avg_similarity
            gates["semantic_similarity_threshold"]["passed"] = avg_similarity >= 0.7

        # Check LLM judge gate
        if "llm_judge" in results:
            llm_scores = [r.get("overall_score", 0) for r in results["llm_judge"]["results"]]
            if llm_scores:
                avg_llm_score = sum(llm_scores) / len(llm_scores)
                gates["llm_judge_threshold"]["actual_value"] = avg_llm_score
                gates["llm_judge_threshold"]["passed"] = avg_llm_score >= 3.5

        return gates

    def _store_evaluation_results(self, report: Dict):
        """Store evaluation results for tracking and analysis"""
        # Store in database, file, or monitoring system
        self.evaluation_history.append(report)

        # Export to CSV for analysis
        self._export_to_csv(report)

        # Send to monitoring/alerting system if quality gates fail
        if not all(gate["passed"] for gate in report["quality_gates"].values()):
            self._send_quality_alert(report)

    def _export_to_csv(self, report: Dict):
        """Export results to CSV for analysis"""
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"evaluation_results_{timestamp}.csv"

        # Flatten results for CSV export
        rows = []
        for method, results in report["detailed_results"].items():
            if isinstance(results.get("results"), list):
                for result in results["results"]:
                    row = {
                        "timestamp": report["evaluation_metadata"]["timestamp"],
                        "method": method,
                        "execution_time": results.get("execution_time", 0),
                        **self._flatten_result(result)
                    }
                    rows.append(row)

        if rows:
            df = pd.DataFrame(rows)
            df.to_csv(filename, index=False)
            self.logger.info(f"📊 Results exported to {filename}")

# Example usage
config = {
    "llm_batch_size": 3,
    "semantic_model": "all-MiniLM-L6-v2",
    "quality_thresholds": {
        "semantic_similarity": 0.7,
        "llm_judge_score": 3.5
    }
}

pipeline = ProductionEvaluationPipeline(config)

# Example test dataset
test_dataset = [
    {
        "id": "test_001",
        "question": "How do I reset my password?",
        "generated_response": "Go to settings and click forgot password to reset your password.",
        "reference_response": "To reset your password: 1) Go to login page 2) Click 'Forgot Password' 3) Enter email 4) Check email for reset link"
    },
    {
        "id": "test_002", 
        "question": "What are your business hours?",
        "generated_response": "We're open Monday through Friday 9 AM to 6 PM.",
        "reference_response": "Our business hours are Monday-Friday 9:00 AM to 6:00 PM, and weekends 10:00 AM to 4:00 PM."
    }
]

# Run comprehensive evaluation
# evaluation_report = await pipeline.run_comprehensive_evaluation(test_dataset)

class CompleteLLMEvaluationSystem:
    """
    A comprehensive evaluation system combining all techniques we've learned
    """

    def __init__(self):
        print("🚀 Initializing Complete LLM Evaluation System")
        self.setup_components()

    def setup_components(self):
        """Initialize all evaluation components"""
        # Core evaluators
        self.semantic_evaluator = AdvancedSemanticEvaluator()
        self.llm_judge = LLMJudge()
        self.context_evaluator = ContextAwareEvaluator()

        # Metrics trackers
        self.evaluation_history = []
        self.performance_trends = {}

        print("✅ All evaluation components initialized")

    def create_evaluation_project(self, project_name: str, domain: str) -> Dict:
        """
        Create a complete evaluation project for a specific domain
        """
        print(f"🎯 Creating evaluation project: {project_name} (Domain: {domain})")

        project = {
            "name": project_name,
            "domain": domain,
            "created_at": datetime.now(),
            "ground_truth": [],
            "test_cases": [],
            "evaluation_results": [],
            "performance_metrics": {},
            "insights": []
        }

        # Generate domain-specific test cases
        project["test_cases"] = self._generate_domain_test_cases(domain)

        # Create ground truth dataset
        project["ground_truth"] = self._create_ground_truth_dataset(domain)

        print(f"✅ Project created with {len(project['test_cases'])} test cases")

        return project

    def _generate_domain_test_cases(self, domain: str) -> List[Dict]:
        """Generate test cases specific to the domain"""
        domain_scenarios = {
            "customer_service": [
                {
                    "scenario": "Password reset request",
                    "question": "I can't remember my password and need to reset it",
                    "context": "frustrated_user",
                    "expected_elements": ["empathy", "clear_steps", "alternatives"]
                },
                {
                    "scenario": "Billing inquiry", 
                    "question": "Why was I charged twice for my subscription?",
                    "context": "confused_user",
                    "expected_elements": ["investigation_offer", "apology", "resolution_timeline"]
                }
            ],
            "technical_support": [
                {
                    "scenario": "Software troubleshooting",
                    "question": "My application keeps crashing when I try to open large files",
                    "context": "technical_user",
                    "expected_elements": ["diagnostic_questions", "step_by_step_guide", "escalation_path"]
                }
            ]
        }

        return domain_scenarios.get(domain, [])

    def run_comprehensive_project_evaluation(self, project: Dict, 
                                           ai_responses: List[str]) -> Dict:
        """
        Run complete evaluation for a project
        """
        print(f"🔬 Running comprehensive evaluation for project: {project['name']}")
        print("=" * 60)

        # Prepare evaluation data
        questions = [tc["question"] for tc in project["test_cases"]]
        reference_answers = [gt["reference_answer"] for gt in project["ground_truth"]]

        # Run all evaluation methods
        results = {}

        # 1. Semantic Evaluation
        print("\n1⃣ SEMANTIC SIMILARITY EVALUATION")
        semantic_results = self.semantic_evaluator.evaluate_semantic_similarity(
            ai_responses, reference_answers
        )
        results["semantic"] = semantic_results

        # 2. ROUGE Evaluation
        print("\n2⃣ ROUGE CONTENT OVERLAP EVALUATION")
        rouge_results = comprehensive_rouge_evaluation(ai_responses, reference_answers)
        results["rouge"] = rouge_results

        # 3. LLM Judge Evaluation
        print("\n3⃣ LLM JUDGE EVALUATION")
        llm_test_cases = [
            {"question": q, "response": r, "id": f"case_{i}"}
            for i, (q, r) in enumerate(zip(questions, ai_responses))
        ]
        llm_results = self.llm_judge.batch_evaluate(llm_test_cases)
        results["llm_judge"] = llm_results

        # 4. Domain-specific Analysis
        print("\n4⃣ DOMAIN-SPECIFIC ANALYSIS")
        domain_results = self._analyze_domain_requirements(
            project, ai_responses
        )
        results["domain_analysis"] = domain_results

        # 5. Generate Final Report
        print("\n5⃣ GENERATING COMPREHENSIVE REPORT")
        final_report = self._generate_project_report(project, results)

        # Store results in project
        project["evaluation_results"].append(final_report)

        return final_report

    def _analyze_domain_requirements(self, project: Dict, 
                                   ai_responses: List[str]) -> Dict:
        """
        Analyze responses against domain-specific requirements
        """
        domain = project["domain"]
        test_cases = project["test_cases"]

        domain_analysis = {
            "domain": domain,
            "requirement_compliance": [],
            "domain_score": 0.0
        }

        for i, (test_case, response) in enumerate(zip(test_cases, ai_responses)):
            compliance = {
                "test_case_id": i,
                "scenario": test_case["scenario"],
                "expected_elements": test_case["expected_elements"],
                "found_elements": [],
                "compliance_score": 0.0
            }

            # Check for expected elements in the response
            response_lower = response.lower()

            for element in test_case["expected_elements"]:
                element_found = self._check_element_presence(element, response_lower)
                if element_found:
                    compliance["found_elements"].append(element)

            # Calculate compliance score
            compliance["compliance_score"] = (
                len(compliance["found_elements"]) / 
                len(test_case["expected_elements"])
            )

            domain_analysis["requirement_compliance"].append(compliance)

        # Calculate overall domain score
        if domain_analysis["requirement_compliance"]:
            domain_analysis["domain_score"] = np.mean([
                c["compliance_score"] 
                for c in domain_analysis["requirement_compliance"]
            ])

        return domain_analysis

    def _check_element_presence(self, element: str, response: str) -> bool:
        """
        Check if a required element is present in the response
        """
        element_indicators = {
            "empathy": ["understand", "sorry", "frustrating", "apologize"],
            "clear_steps": ["step", "first", "then", "next", "1.", "2."],
            "alternatives": ["also", "alternatively", "or", "another option"],
            "investigation_offer": ["investigate", "look into", "check", "review"],
            "apology": ["sorry", "apologize", "regret", "mistake"],
            "diagnostic_questions": ["can you", "what happens", "when did", "which"],
            "escalation_path": ["escalate", "supervisor", "specialist", "technical team"]
        }

        indicators = element_indicators.get(element, [element.replace("_", " ")])
        return any(indicator in response for indicator in indicators)

    def _generate_project_report(self, project: Dict, results: Dict) -> Dict:
        """
        Generate comprehensive project evaluation report
        """
        report = {
            "project_name": project["name"],
            "evaluation_timestamp": datetime.now().isoformat(),
            "overall_scores": {},
            "detailed_results": results,
            "insights": [],
            "recommendations": [],
            "quality_assessment": "needs_evaluation"
        }

        # Calculate overall scores
        if "semantic" in results:
            report["overall_scores"]["semantic_similarity"] = results["semantic"]["average_similarity"]

        if "rouge" in results:
            report["overall_scores"]["content_overlap"] = results["rouge"]["averages"]["rouge1"]

        if "llm_judge" in results:
            llm_scores = [r.get("overall_score", 0) for r in results["llm_judge"] if "overall_score" in r]
            if llm_scores:
                report["overall_scores"]["llm_judge_score"] = sum(llm_scores) / len(llm_scores)

        if "domain_analysis" in results:
            report["overall_scores"]["domain_compliance"] = results["domain_analysis"]["domain_score"]

        # Generate insights and recommendations
        report["insights"] = self._generate_insights(results)
        report["recommendations"] = self._generate_recommendations(results)

        # Overall quality assessment
        report["quality_assessment"] = self._assess_overall_quality(report["overall_scores"])

        print("\n📊 FINAL EVALUATION SUMMARY")
        print("=" * 50)
        for metric, score in report["overall_scores"].items():
            print(f"   {metric}: {score:.3f}")
        print(f"\n🏆 Overall Quality: {report['quality_assessment']}")

        return report

# Example: Complete evaluation project
evaluation_system = CompleteLLMEvaluationSystem()

# Create a customer service evaluation project
cs_project = evaluation_system.create_evaluation_project(
    "Customer Service Bot Evaluation",
    "customer_service"
)

# Example AI responses to evaluate
ai_responses = [
    "I understand this is frustrating. To reset your password, please go to the login page and click 'Forgot Password'. You can also contact support if you need additional help.",
    "I apologize for the double charge. Let me investigate this immediately and check your billing history. I'll make sure to resolve this within 24 hours and process any necessary refunds."
]

# Run comprehensive evaluation
final_report = evaluation_system.run_comprehensive_project_evaluation(
    cs_project, ai_responses
)

def select_evaluation_strategy(use_case: str, constraints: Dict) -> List[str]:
    """
    Smart evaluation strategy selection based on use case and constraints
    """

    strategies = {
        "quick_prototype": ["hit_rate", "basic_similarity"],
        "production_launch": ["hit_rate", "mrr", "rouge", "llm_judge", "semantic_similarity"],
        "cost_sensitive": ["hit_rate", "mrr", "rouge"],
        "quality_critical": ["llm_judge", "semantic_similarity", "human_evaluation"],
        "real_time": ["hit_rate", "basic_similarity"],
        "research": ["all_metrics", "human_evaluation", "statistical_significance"]
    }

    recommended = strategies.get(use_case, ["hit_rate", "mrr", "rouge"])

    # Adjust based on constraints
    if constraints.get("budget", "medium") == "low":
        recommended = [m for m in recommended if m not in ["llm_judge", "human_evaluation"]]

    if constraints.get("latency", "medium") == "low":
        recommended = [m for m in recommended if m in ["hit_rate", "mrr"]]

    return recommended

# Example usage
strategy = select_evaluation_strategy(
    "production_launch", 
    {"budget": "medium", "latency": "medium"}
)
print(f"Recommended evaluation strategy: {strategy}")

# ❌ Don't do this - optimizing for metric instead of real performance
def bad_system_optimization():
    # System learns to game ROUGE scores by repeating reference text
    return "This is exactly the reference answer repeated word for word"

# ✅ Do this - use multiple diverse metrics
def good_evaluation_approach():
    metrics = [
        "semantic_similarity",  # Catches meaning preservation
        "rouge_scores",        # Catches content coverage
        "llm_judge",          # Catches overall quality
        "human_evaluation"    # Catches real-world usefulness
    ]
    return metrics

# ❌ Don't do this - limited test cases
bad_ground_truth = [
    {"question": "How do I reset password?", "answer": "..."},
    {"question": "How to reset password?", "answer": "..."},
    {"question": "Password reset help?", "answer": "..."}
]

# ✅ Do this - diverse, comprehensive test cases
good_ground_truth = [
    {"question": "How do I reset password?", "difficulty": "easy", "user_type": "beginner"},
    {"question": "I'm locked out and the reset email isn't working", "difficulty": "hard", "user_type": "frustrated"},
    {"question": "Can you walk me through account recovery?", "difficulty": "medium", "user_type": "polite"},
    {"question": "HELP! Can't login!!!", "difficulty": "medium", "user_type": "urgent"}
]

class EvaluationMonitor:
    """
    Monitor evaluation trends and detect performance drift
    """

    def __init__(self):
        self.historical_scores = []
        self.alerts = []

    def track_evaluation_trends(self, new_scores: Dict):
        """
        Track evaluation scores over time and detect trends
        """
        self.historical_scores.append({
            "timestamp": datetime.now(),
            "scores": new_scores
        })

        # Detect significant changes
        self._detect_performance_drift()
        self._check_quality_regression()

    def _detect_performance_drift(self):
        """Detect gradual performance degradation"""
        if len(self.historical_scores) < 5:
            return

        recent_scores = self.historical_scores[-5:]
        for metric in recent_scores[0]["scores"].keys():
            scores = [s["scores"][metric] for s in recent_scores]

            # Simple trend detection
            if self._is_declining_trend(scores):
                self.alerts.append({
                    "type": "performance_drift",
                    "metric": metric,
                    "severity": "warning",
                    "message": f"Declining trend detected in {metric}"
                })

    def _is_declining_trend(self, scores: List[float]) -> bool:
        """Simple trend detection"""
        if len(scores) < 3:
            return False

        # Check if last 3 scores are consistently declining
        return scores[-1] < scores[-2] < scores[-3]

# Example usage
monitor = EvaluationMonitor()

# Simulate tracking scores over time
for week in range(10):
    # Simulate gradually declining performance
    base_score = 0.8 - (week * 0.02)  
    weekly_scores = {
        "semantic_similarity": base_score + np.random.normal(0, 0.05),
        "llm_judge_score": (base_score * 5) + np.random.normal(0, 0.2)
    }
    monitor.track_evaluation_trends(weekly_scores)

print(f"Alerts generated: {len(monitor.alerts)}")
for alert in monitor.alerts:
    print(f"⚠ {alert['type']}: {alert['message']}")