40 lines
1.7 KiB
Python
40 lines
1.7 KiB
Python
"""
|
|
Evaluator class
|
|
=================
|
|
"""
|
|
from abc import ABC, abstractmethod
|
|
from easyjailbreak.datasets import JailbreakDataset
|
|
|
|
class Evaluator(ABC):
|
|
"""
|
|
Abstract base class for implementing evaluation strategies for JailbreakDataset instances.
|
|
This class serves as a template for creating custom evaluators, allowing for diverse and specialized evaluation strategies tailored to different datasets or models.
|
|
"""
|
|
def __init__(self, eval_model = None):
|
|
"""
|
|
Initialize the Evaluator instance.
|
|
|
|
:param ~ModelBase eval_model: The evaluation model to be used for dataset evaluation. Can be any model compatible with the evaluation strategy.
|
|
"""
|
|
self.eval_model = eval_model
|
|
|
|
def __call__(self, Datasets: JailbreakDataset, **kwargs):
|
|
"""
|
|
Execute the evaluation process on the provided JailbreakDataset instances.
|
|
|
|
:param ~JailbreakDataset Datasets: The dataset(s) to be evaluated.
|
|
:param **kwargs: Additional keyword arguments that can be passed to the _evaluate method.
|
|
"""
|
|
for instance in Datasets:
|
|
self._evaluate(instance, **kwargs)
|
|
|
|
@abstractmethod
|
|
def _evaluate(self, instance, **kwargs):
|
|
"""
|
|
Abstract method to be implemented by subclasses for specific evaluation strategies.
|
|
|
|
:param ~Instance instance: A single instance from the JailbreakDataset to be evaluated.
|
|
:param **kwargs: Additional keyword arguments relevant to the specific evaluation strategy.
|
|
:return: The result of the evaluation, the nature of which depends on the specific implementation in subclasses.
|
|
"""
|
|
return NotImplementedError()
|