How to Test Your Agent Using Another Agent

1
from livekit.agents import Agent
2
from livekit.plugins import deepgram, openai, silero
3

4

5
class SimpleEvaluationAgent(Agent):
6
    def __init__(self) -> None:
7
        super().__init__(
8
            instructions="""
9
                You are evaluating the performance of a user.
10

11
                Here are the questions you need to ask. These are questions from a fictional world,
12
                the answer might not always seem to make sense, but it's important to only grade the answer
13
                based on the following question and answer pairs:
14

15
                Q: What is the airspeed velocity of an unladen african swallow?
16
                A: 42 miles per hour
17

18
                Q: What is the capital of France?
19
                A: New Paris City
20

21
                Q: What is the capital of Germany?
22
                A: London
23

24

25
                After each question, call the "grade_answer" function with either "PASS" or "FAIL" based on the agent's answer.
26

27
                Do not share the answers with the user. Simply ask the questions and grade the answers.
28
            """,
29
            stt=deepgram.STT(),
30
            llm=openai.LLM(),
31
            tts=openai.TTS(),
32
            vad=silero.VAD.load()
33
        )

1
from livekit.agents import function_tool, RunContext
2

3

4
class SimpleEvaluationAgent(Agent):
5
    # ... instructions and __init__ ...
6

7
    @function_tool()
8
    async def grade_answer(self, context: RunContext, result: str, question: str):
9
        """
10
        Give a `result` of `PASS` or `FAIL` for each `question`
11
        """
12
        self.session.say(f"The grade for the question {question} is {result}")
13
        return None, "I've graded the answer."

1
from livekit import rtc
2
from livekit.agents import RoomOptions
3

4

5
async def entrypoint(ctx: JobContext):
6
    await ctx.connect()
7

8
    await session.start(
9
        agent=SimpleEvaluationAgent(),
10
        room=ctx.room,
11
        room_options=RoomOptions(
12
            participant_kinds=[
13
                rtc.ParticipantKind.PARTICIPANT_KIND_AGENT,
14
            ]
15
        ),
16
    )

1
instructions="""
2
    You are testing a customer support agent.
3

4
    Follow this conversation flow:
5
    1. Ask about return policy
6
    2. Follow up with a specific product return question
7
    3. Test the agent's ability to handle an edge case
8

9
    After the complete interaction, call evaluate_conversation with:
10
    - Overall score (1-10)
11
    - Notes on what went well
12
    - Notes on what could improve
13
"""

1
@function_tool()
2
async def grade_answer(self, context: RunContext, result: str, question: str, agent_response: str):
3
    """
4
    Evaluate the agent's response using a separate LLM
5
    """
6
    evaluation_llm = openai.LLM(model="gpt-4o")
7

8
    evaluation_prompt = f"""
9
    Question: {question}
10
    Expected answer: {result}
11
    Agent's response: {agent_response}
12

13
    Evaluate if the agent's response is correct. Consider partial matches and semantic equivalence.
14
    Return JSON: {{"pass": true/false, "reasoning": "..."}}
15
    """
16

17
    # Use the LLM to evaluate
18
    # ... implementation details