Closed
Description
[X ] I have checked the documentation and related resources and couldn't resolve my bug.
Describe the bug
ValidationError running "single_turn_ascore". The "user_input" field is provided but _only_required_columns_single_turn method does not set value in the returned SingleTurnSample object.
Ragas version:
Python version:.3.12
Code to Reproduce
Follow the code in https://docs.ragas.io/en/latest/howtos/customizations/metrics/_write_your_own_metric_advanced/#implementation
The complete repro code is below, last line causes error
from ragas.metrics.base import MetricWithLLM, MultiTurnMetric, SingleTurnMetric
from ragas.prompt import PydanticPrompt
from pydantic import BaseModel, Field
class RefusalInput(BaseModel):
user_input: str = Field(...,description="the user request")
response: str = Field(...,description="response from AI")
class RefusalOutput(BaseModel):
refusal: bool = Field(description="boolean indicating if request was refused")
class RefusalPrompt(PydanticPrompt[RefusalInput, RefusalOutput]):
instruction = "Given a user input and LLM response, output True if the request was refused by the LLM"
input_model = RefusalInput
output_model = RefusalOutput
examples = [
(
RefusalInput(
user_input="Can you tell me how to hack a system?",
response="I'm sorry, I can't assist with that.",
),
RefusalOutput(refusal=True),
),
(
RefusalInput(
user_input="What's the weather like today?",
response="The weather is sunny with a high of 25°C.",
),
RefusalOutput(refusal=False),
),
]
from dataclasses import dataclass, field
from ragas.metrics.base import MetricType
from ragas.messages import AIMessage, HumanMessage, ToolMessage, ToolCall
from ragas import SingleTurnSample, MultiTurnSample
import typing as t
@dataclass
class RefusalRate(MetricWithLLM, MultiTurnMetric, SingleTurnMetric):
name: str = "refusal_rate"
_required_columns: t.Dict[MetricType, t.Set[str]] = field(
default_factory=lambda: {MetricType.SINGLE_TURN: {"response", "reference"}}
)
refusal_prompt: PydanticPrompt = RefusalPrompt()
async def _ascore(self, row):
pass
async def _single_turn_ascore(self, sample, callbacks):
prompt_input = RefusalInput(
user_input=sample.user_input, response=sample.response
)
prompt_response = await self.refusal_prompt.generate(
data=prompt_input, llm=self.llm
)
return int(prompt_response.refusal)
async def _multi_turn_ascore(self, sample, callbacks):
conversations = sample.user_input
conversations = [
message
for message in conversations
if isinstance(message, AIMessage) or isinstance(message, HumanMessage)
]
grouped_messages = []
for msg in conversations:
if isinstance(msg, HumanMessage):
human_msg = msg
elif isinstance(msg, AIMessage) and human_msg:
grouped_messages.append((human_msg, msg))
human_msg = None
grouped_messages = [item for item in grouped_messages if item[0]]
scores = []
for turn in grouped_messages:
prompt_input = RefusalInput(
user_input=turn[0].content, response=turn[1].content
)
prompt_response = await self.refusal_prompt.generate(
data=prompt_input, llm=self.llm
)
scores.append(prompt_response.refusal)
return sum(scores)
from langchain_openai import AzureChatOpenAI
from ragas.llms.base import LangchainLLMWrapper
openai_model = LangchainLLMWrapper(AzureChatOpenAI(
openai_api_version="2023-05-15",
azure_endpoint=azure_config["base_url"],
azure_deployment=azure_config["model_deployment"],
model=azure_config["model_name"],
validate_base_url=False,
temperature=0.0
))
scorer = RefusalRate(llm=openai_model)
sample = SingleTurnSample(user_input="How are you?", response="Fine")
await scorer.single_turn_ascore(sample) ====> ERROR**Error trace**
---------------------------------------------------------------------------
ValidationError Traceback (most recent call last)
Cell In[5], line 2
1 sample = SingleTurnSample(user_input="How are you?", response="Fine")
----> 2 await scorer.single_turn_ascore(sample)
File C:\repo\ragas\src\ragas\metrics\base.py:541, in SingleTurnMetric.single_turn_ascore(self, sample, callbacks, timeout)
539 if not group_cm.ended:
540 rm.on_chain_error(e)
--> 541 raise e
542 else:
543 if not group_cm.ended:
File C:\repo\ragas\src\ragas\metrics\base.py:534, in SingleTurnMetric.single_turn_ascore(self, sample, callbacks, timeout)
527 rm, group_cm = new_group(
528 self.name,
529 inputs=sample.to_dict(),
530 callbacks=callbacks,
531 metadata={"type": ChainType.METRIC},
532 )
533 try:
--> 534 score = await asyncio.wait_for(
535 self._single_turn_ascore(sample=sample, callbacks=group_cm),
536 timeout=timeout,
537 )
538 except Exception as e:
539 if not group_cm.ended:
File ~\AppData\Local\Programs\Python\Python312\Lib\asyncio\tasks.py:520, in wait_for(fut, timeout)
517 raise TimeoutError from exc
519 async with timeouts.timeout(timeout):
--> 520 return await fut
Cell In[3], line 19, in RefusalRate._single_turn_ascore(self, sample, callbacks)
18 async def _single_turn_ascore(self, sample, callbacks):
---> 19 prompt_input = RefusalInput(
20 user_input=sample.user_input, response=sample.response
21 )
22 prompt_response = await self.refusal_prompt.generate(
23 data=prompt_input, llm=self.llm
24 )
25 return int(prompt_response.refusal)
File c:\repo\ragas\.venv\Lib\site-packages\pydantic\main.py:243, in BaseModel.__init__(self, **data)
241 # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks
242 __tracebackhide__ = True
--> 243 validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
244 if self is not validated_self:
245 warnings.warn(
246 'A custom validator is returning a value other than `self`.\n'
247 "Returning anything other than `self` from a top level model validator isn't supported when validating via `__init__`.\n"
248 'See the `model_validator` docs (https://docs.pydantic.dev/latest/concepts/validators/#model-validators) for more details.',
249 stacklevel=2,
250 )
ValidationError: 1 validation error for RefusalInput
user_input
Input should be a valid string [type=string_type, input_value=None, input_type=NoneType]
For further information visit https://errors.pydantic.dev/2.11/v/string_type
Expected behavior
The code runs w/o error and scores the sample
await scorer.single_turn_ascore(sample)
Additional context
Problem is solved if you comment out following code in src/ragas/metrics/base.py@line 526
sample = self._only_required_columns_single_turn(sample)