Model Configuration

1
import os
2
3
from nemo_evaluator.sdk import Evaluator
4
from nemo_platform import NeMoPlatform
5
6
7
client = NeMoPlatform(
8
    base_url=os.environ.get("NMP_BASE_URL", "http://localhost:8080"),
9
    workspace="default",
10
)
11
evaluator: Evaluator = client.evaluator  # this object is an Evaluator resource

1
from nemo_evaluator_sdk import Model
2
3
4
model = Model(
5
    url="https://integrate.api.nvidia.com/v1",
6
    name="meta/llama-3.1-70b-instruct",
7
    format="nim",
8
    api_key_secret="NVIDIA_API_KEY",
9
)

1
client.secrets.create(
2
    name="nvidia-api-key",
3
    value=os.environ["NVIDIA_API_KEY"],
4
)

1
from nemo_evaluator_sdk import (
2
    RunConfigOnlineModel,
3
    ExactMatchMetric,
4
    InferenceParams,
5
    Model,
6
)
7
8
9
model = Model(
10
    url="https://integrate.api.nvidia.com/v1",
11
    name="meta/llama-3.1-70b-instruct",
12
    format="nim",
13
    api_key_secret="NVIDIA_API_KEY",
14
)
15
16
metric = ExactMatchMetric(reference="{{item.expected_answer}}")
17
18
result = evaluator.run(
19
    metric=metric,
20
    dataset=[
21
        {"question": "What is the capital of France?", "expected_answer": "Paris"},
22
    ],
23
    config=RunConfigOnlineModel(
24
        parallelism=4,
25
        inference=InferenceParams(temperature=0.1, max_tokens=64),
26
    ),
27
    target=model,
28
    prompt_template="Answer this question concisely: {{item.question}}",
29
)

1
from nemo_evaluator_sdk import Model, RangeScore, LLMJudgeMetric
2
3
judge_model = Model(
4
    url="https://integrate.api.nvidia.com/v1",
5
    name="meta/llama-3.1-70b-instruct",
6
    format="nim",
7
    api_key_secret="NVIDIA_API_KEY",
8
)
9
metric = LLMJudgeMetric(
10
    model=judge_model,
11
    scores=[
12
        RangeScore(
13
            name="correctness",
14
            description="Correctness from 1 to 5.",
15
            minimum=1,
16
            maximum=5,
17
        ),
18
    ],
19
    prompt_template={
20
        "messages": [
21
            {
22
                "role": "system",
23
                "content": "Return JSON with a correctness score from 1 to 5.",
24
            },
25
            {
26
                "role": "user",
27
                "content": "Question: {{item.question}}\nAnswer: {{item.output}}\nExpected: {{item.expected_answer}}",
28
            },
29
        ],
30
    },
31
)
32
33
result = evaluator.run(
34
    metric=metric,
35
    dataset=[
36
        {
37
            "question": "What is the capital of France?",
38
            "output": "Paris",
39
            "expected_answer": "Paris",
40
        },
41
    ],
42
)

1
from nemo_evaluator_sdk import (
2
    RunConfigOnlineModel,
3
    InferenceParams,
4
    ReasoningParams,
5
)
6
7
8
params = RunConfigOnlineModel(
9
    parallelism=4,
10
    request_timeout=60,
11
    max_retries=2,
12
    ignore_request_failure=False,
13
    inference=InferenceParams(temperature=0.2, max_tokens=256),
14
    reasoning=ReasoningParams(end_token="</think>"),
15
)