Agent Configuration

1
import os
2
3
from nemo_evaluator.sdk import Evaluator
4
from nemo_platform import NeMoPlatform
5
6
7
client = NeMoPlatform(
8
    base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"),
9
    workspace="default",
10
)
11
evaluator: Evaluator = client.evaluator  # this object is an Evaluator resource

1
client.secrets.create(
2
    name="my-agent-api-key",
3
    value=os.environ["MY_AGENT_API_KEY"],
4
)

1
from nemo_evaluator_sdk import Agent, RunConfigOnline
2
from nemo_evaluator_sdk import ExactMatchMetric
3
metric = ExactMatchMetric(reference="{{item.expected_answer}}")
4
agent = Agent(
5
    url="https://my-agent.example.com/invoke",
6
    name="qa-agent",
7
    format="generic",
8
    api_key_secret="MY_AGENT_API_KEY",
9
    body={"question": "{{ prompt }}"},
10
    response_path="$.answer",
11
    trajectory_path="$.reasoning_steps",
12
)
13
14
result = evaluator.run(
15
    metric=metric,
16
    dataset=[
17
        {"question": "What is the capital of France?", "expected_answer": "Paris"},
18
    ],
19
    config=RunConfigOnline(parallelism=4, request_timeout=60, max_retries=2),
20
    target=agent,
21
    prompt_template="Question: {{item.question}}\nAnswer:",
22
)
23
for score in result.aggregate_scores.scores:
24
    print(f"{score.name}: mean={score.mean}")

1
from fastapi import FastAPI
2
from pydantic import BaseModel
3
4
app = FastAPI()
5
6
7
class AgentRequest(BaseModel):
8
    question: str
9
10
11
class AgentResponse(BaseModel):
12
    answer: str
13
    reasoning_steps: list[dict]
14
15
16
@app.post("/invoke")
17
async def invoke(request: AgentRequest) -> AgentResponse:
18
    return AgentResponse(
19
        answer="Paris",
20
        reasoning_steps=[
21
            {"step": "search", "result": "Found relevant documents"},
22
            {"step": "synthesize", "result": "Generated answer from context"},
23
        ],
24
    )

1
from nemo_evaluator_sdk import Agent, RunConfigOnline
2
3
4
from nemo_evaluator_sdk import ExactMatchMetric
5
metric = ExactMatchMetric(reference="{{item.expected_answer}}")
6
agent = Agent(
7
    url="https://my-nat-agent.example.com",
8
    name="nat-research-agent",
9
    format="nemo_agent_toolkit",
10
    api_key_secret="my-nat-api-key",
11
)
12
13
job = evaluator.submit(
14
    metric=metric,
15
    dataset=[
16
        {"question": "What is the capital of France?", "expected_answer": "Paris"},
17
    ],
18
    config=RunConfigOnline(parallelism=4),
19
    target=agent,
20
    prompt_template={
21
        "messages": [
22
            {"role": "user", "content": "{{item.question}}"},
23
        ],
24
    },
25
)
26
job.wait_until_done()
27
result = job.get_result()