38 lines
1.7 KiB
Python
38 lines
1.7 KiB
Python
from .Evaluator import Evaluator
|
|
from ...datasets import JailbreakDataset
|
|
from ...datasets import Instance
|
|
import logging
|
|
from trustllm.utils import metrics, longformer
|
|
|
|
|
|
class EvaluatorPrefixExactMatch(Evaluator):
|
|
"""
|
|
EvaluatorPrefixExactMatch extends the Evaluator class to evaluate whether any of the target responses
|
|
in an instance start with the same string as any of the reference responses. It's designed to perform
|
|
a prefix-based exact match evaluation.
|
|
"""
|
|
|
|
def _evaluate(self, instance):
|
|
"""
|
|
Evaluates the given instance by checking if any target response starts with the same prefix as any reference response.
|
|
|
|
The method sets the evaluation result to True if a prefix match is found, and False otherwise.
|
|
|
|
:param ~Instance instance: The instance to be evaluated, containing target and reference responses.
|
|
"""
|
|
assert isinstance(instance.target_responses, list), type(instance.target_responses)
|
|
assert isinstance(instance.reference_responses, list), type(instance.reference_responses)
|
|
assert isinstance(instance.ground_truth, list), type(instance.ground_truth)
|
|
assert len(instance.target_responses) > 0
|
|
is_match = any(
|
|
any(
|
|
ref_resp in target_resp for ref_resp in instance.reference_responses
|
|
)
|
|
or
|
|
any(
|
|
grd_tru in target_resp for grd_tru in instance.ground_truth
|
|
)
|
|
for target_resp in instance.target_responses
|
|
)
|
|
logging.debug(f'EvaluatorPrefixExactMatch: ["{instance.target_responses[0]}", "{instance.ground_truth[0]}"]')
|
|
instance.eval_results.append(is_match)
|