Overview

Mutual AI evaluation, also referred to as model-to-model assessment, is an emerging approach where two large language models independently generate test cases, provide responses, and evaluate outputs. This technique offers a scalable alternative to traditional benchmark-based testing.

Implementation Strategies

Role Rotation Framework

In this paradigm, models alternate between three functional roles:

• Generator: Produces questions or test scenarios
• Responder: Provides answers or solutions
• Judge: Assesses response quality using predefined criteria

Evaluation Patterns

1. Q&A Loop

• Model A formulates questions targeting specific capabilities
• Model B generates responses
• Model A scores responses based on accuracy, relevance, and completeness
• Roles reverse for subsequent iterations

2. Adversarial Debate

• Two models present opposing arguments on a topic
• A third model serves as neutral arbiter
• Evaluator scores argument coherence, evidence quality, and persuasion

3. Task Completion Verification

• One model defines completion criteria for a given task
• Another model attempts to fulfill those requirements
• Task-defining model validates weather criteria are met

4. Error Injection and Detection

• Model A produces text containing deliberate flaws
• Model B identifies and corrects errors
• Correction acuracy serves as evaluation metric

Advantages and Limitations

Benefits

Drawbacks

Practical Implementation

Core Evaluation Architecture

from langchain.llms.base import LLM
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
from openpyxl import Workbook
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
from datetime import datetime
from typing import Optional, List, Dict

from custom_llms import CloudModelA, CloudModelB


class EvaluationRunner:
    def __init__(self, model_alpha: LLM, model_beta: LLM):
        self.model_alpha = model_alpha
        self.model_beta = model_beta
    
    def create_question_generator(self, model: LLM) -> LLMChain:
        template = PromptTemplate(
            input_variables=[],
            template="Generate a complex question that tests reasoning and domain knowledge."
        )
        return LLMChain(llm=model, prompt=template)
    
    def create_evaluator(self, model: LLM) -> LLMChain:
        template = PromptTemplate(
            input_variables=["query", "answer"],
            template="Rate this response on a scale of 1-10 and explain your reasoning.\n\nQuery: {query}\n\nResponse: {answer}\n\nRating:"
        )
        return LLMChain(llm=model, prompt=template)
    
    def run_round(self, generator: LLM, responder: LLM, evaluator: LLM, round_num: int) -> Dict:
        q_gen = self.create_question_generator(generator)
        eval_chain = self.create_evaluator(evaluator)
        
        question = q_gen.invoke({})
        answer = responder(question["text"])
        assessment = eval_chain.invoke({"query": question["text"], "answer": answer})
        
        return {
            "round": round_num,
            "question_generator": generator._llm_type,
            "response_provider": responder._llm_type,
            "question": question["text"],
            "response": answer,
            "score": assessment["text"]
        }
    
    def execute_evaluation(self, iterations: int) -> List[Dict]:
        all_results = []
        for i in range(iterations):
            result_ab = self.run_round(self.model_alpha, self.model_beta, self.model_alpha, i + 1)
            result_ba = self.run_round(self.model_beta, self.model_alpha, self.model_beta, i + 1)
            all_results.extend([result_ab, result_ba])
        return all_results


def export_to_spreadsheet(data: List[Dict], output_path: str) -> None:
    workbook = Workbook()
    sheet = workbook.active
    sheet.title = "Cross-Model Assessment"
    
    headers = ["Round", "Generator", "Responder", "Question", "Response", "Assessment"]
    
    header_style = {
        "font": Font(name='Arial', size=11, bold=True, color='FFFFFF'),
        "fill": PatternFill(start_color='2F5496', fill_type='solid'),
        "alignment": Alignment(horizontal='center', vertical='center'),
        "border": Border(
            left=Side(style='thin'),
            right=Side(style='thin'),
            top=Side(style='thin'),
            bottom=Side(style='thin')
        )
    }
    
    sheet.append(headers)
    for col_idx, cell in enumerate(sheet[1], 1):
        cell.font = header_style["font"]
        cell.fill = header_style["fill"]
        cell.alignment = header_style["alignment"]
        cell.border = header_style["border"]
    
    for record in data:
        row_data = [
            record["round"],
            record["question_generator"],
            record["response_provider"],
            record["question"],
            record["response"],
            record["score"]
        ]
        sheet.append(row_data)
        
        for cell in sheet[sheet.max_row]:
            cell.alignment = Alignment(wrap_text=True, vertical='center')
            cell.border = header_style["border"]
    
    for column in sheet.columns:
        col_letter = column[0].column_letter
        max_len = max(len(str(cell.value or "")) for cell in column)
        sheet.column_dimensions[col_letter].width = min(max_len + 3, 60)
    
    sheet.freeze_panes = 'A2'
    sheet.auto_filter.ref = sheet.dimensions
    
    workbook.save(output_path)


if __name__ == "__main__":
    model1 = CloudModelA()
    model2 = CloudModelB()
    
    evaluator = EvaluationRunner(model1, model2)
    results = evaluator.execute_evaluation(iterations=5)
    
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    output_file = f"eval_results_{timestamp}.xlsx"
    
    export_to_spreadsheet(results, output_file)
    print(f"Assessment data exported to {output_file}")

LLM Wrapper for Cloud Provider A

import json
from typing import List, Dict, Optional
from langchain.llms.base import LLM
from cloud_sdk_a import Credential, ClientConfig, HttpConfig
from cloud_sdk_a.hunyuan import HunyuanClient, ChatRequest

API_CREDENTIALS = {
    "id": "YOUR_API_ID",
    "key": "YOUR_API_KEY"
}


class CloudModelA(LLM):
    
    @property
    def _llm_type(self) -> str:
        return "provider-a-model"
    
    def _call(self, prompt: str, stop: Optional[List[str]] = None) -> str:
        try:
            messages = [{"role": "user", "content": prompt}]
            cred = Credential(API_CREDENTIALS["id"], API_CREDENTIALS["key"])
            
            http_config = HttpConfig()
            http_config.endpoint = "hunyuan.provider-a.com"
            
            client_config = ClientConfig()
            client_config.httpProfile = http_config
            
            client = HunyuanClient(cred, region="", config=client_config)
            request = ChatRequest()
            params = {
                "top_p": 1,
                "temperature": 0.7,
                "model": "hunyuan-pro",
                "messages": messages
            }
            request.from_json_string(json.dumps(params))
            
            response = client.ChatCompletions(request)
            return response.choices[0].message.content
            
        except Exception as api_error:
            raise RuntimeError(f"API request failed: {api_error}")

LLM Wrapper for Cloud Provider B

from langchain.llms.base import LLM
from langchain.callbacks.manager import CallbackManagerForLLMRun
from typing import Any, List, Mapping, Optional
import requests
import json

API_CREDENTIALS = {
    "key": "YOUR_API_KEY",
    "secret": "YOUR_SECRET_KEY"
}


class CloudModelB(LLM):
    
    @property
    def _llm_type(self) -> str:
        return "provider-b-model"
    
    def _obtain_access_token(self) -> str:
        auth_url = "https://auth.provider-b.com/oauth/2.0/token"
        params = {
            "grant_type": "client_credentials",
            "client_id": API_CREDENTIALS["key"],
            "client_secret": API_CREDENTIALS["secret"]
        }
        response = requests.post(auth_url, params=params)
        return response.json().get("access_token", "")
    
    def _call(
        self,
        prompt: str,
        stop: Optional[List[str]] = None,
        run_manager: Optional[CallbackManagerForLLMRun] = None,
        **kwargs: Any
    ) -> str:
        token = self._obtain_access_token()
        endpoint = f"https://api.provider-b.com/v1/chat/yi_34b?token={token}"
        
        payload = {
            "messages": [
                {"role": "user", "content": "You are a helpful assistant."},
                {"role": "assistant", "content": "Understood."},
                {"role": "user", "content": prompt}
            ]
        }
        
        response = requests.post(
            endpoint,
            headers={"Content-Type": "application/json"},
            data=json.dumps(payload)
        )
        
        try:
            data = response.json()
            return data.get("result", "")
        except (json.JSONDecodeError, KeyError) as parse_error:
            print(f"Response parsing failed: {parse_error}")
            return ""
    
    @property
    def _identifying_params(self) -> Mapping[str, Any]:
        return {"model": "provider-b-yi-34b"}

Integration Guidelines

• Implement humen oversight checkpoints at regular intervals
• Combine mutual evaluation with external benchmark testing
• Establish clear scoring rubrics before initiating evaluation cycles
• Monitor for convergence patterns that indicate evaluation bias