Add files via upload

easyjailbreak/metrics/Evaluator/Evaluator.py

@@ -0,0 +1,40 @@
+"""
+Evaluator class
+=================
+"""
+from abc import ABC, abstractmethod
+from easyjailbreak.datasets import JailbreakDataset
+
+class Evaluator(ABC):
+    """
+    Abstract base class for implementing evaluation strategies for JailbreakDataset instances.
+    This class serves as a template for creating custom evaluators, allowing for diverse and specialized evaluation strategies tailored to different datasets or models.
+    """
+    def __init__(self, eval_model = None):
+        """
+        Initialize the Evaluator instance.
+
+        :param ~ModelBase eval_model: The evaluation model to be used for dataset evaluation. Can be any model compatible with the evaluation strategy.
+        """
+        self.eval_model = eval_model
+
+    def __call__(self, Datasets: JailbreakDataset, **kwargs):
+        """
+        Execute the evaluation process on the provided JailbreakDataset instances.
+
+        :param ~JailbreakDataset Datasets: The dataset(s) to be evaluated.
+        :param **kwargs: Additional keyword arguments that can be passed to the _evaluate method.
+        """
+        for instance in Datasets:
+            self._evaluate(instance, **kwargs)
+
+    @abstractmethod
+    def _evaluate(self, instance, **kwargs):
+        """
+        Abstract method to be implemented by subclasses for specific evaluation strategies.
+
+        :param ~Instance instance: A single instance from the JailbreakDataset to be evaluated.
+        :param **kwargs: Additional keyword arguments relevant to the specific evaluation strategy.
+        :return: The result of the evaluation, the nature of which depends on the specific implementation in subclasses.
+        """
+        return NotImplementedError()