Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions giskard/rag/metrics/ragas_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import logging

import ragas

from ...llm.client import ChatMessage, LLMClient, get_default_client
from ...llm.embeddings import BaseEmbedding, get_default_embedding
from ..base import AgentAnswer
Expand Down Expand Up @@ -104,20 +106,38 @@ def __call__(self, question_sample: dict, answer: AgentAnswer) -> dict:

self.metric.init(run_config)
if self.requires_context and answer.documents is None:
logger.warn(
logger.warning(
f"No retrieved documents are passed to the evaluation function, computation of {self.name} cannot be done without it."
"Make sure you pass 'retrieved_documents' to the evaluate function or that the 'answer_fn' return documents alongside the answer."
)
return {self.name: 0}

ragas_sample = {
"question": question_sample["question"],
"answer": answer.message,
"contexts": answer.documents,
"ground_truth": question_sample["reference_answer"],
}
ragas_sample = self.prepare_ragas_sample(question_sample, answer)

return {self.name: self.metric.score(ragas_sample)}

@staticmethod
def prepare_ragas_sample(question_sample: dict, answer: AgentAnswer) -> dict:
if ragas.__version__.startswith("0.1"):
logger.warning(
f"{DeprecationWarning.__name__}: You are using an older version (v0.1) of `ragas` package. "
"Support for v0.1 is deprecated and may be removed in future versions. "
"Please consider updating `ragas` to a later version."
)
return {
"question": question_sample["question"],
"answer": answer.message,
"contexts": answer.documents,
"ground_truth": question_sample["reference_answer"],
}

return {
"user_input": question_sample["question"],
"response": answer.message,
"retrieved_contexts": answer.documents,
"reference": question_sample["reference_answer"],
}


ragas_context_precision = RagasMetric(name="RAGAS Context Precision", metric=context_precision, requires_context=True)
ragas_faithfulness = RagasMetric(name="RAGAS Faithfulness", metric=faithfulness, requires_context=True)
Expand Down