Add files via upload

easyjailbreak/metrics/Evaluator/Evaluator_Match.py

@@ -0,0 +1,38 @@
+"""
+EvalatorMatch class
+=====================
+"""
+from easyjailbreak.metrics.Evaluator import Evaluator
+from easyjailbreak.datasets import Instance
+class EvalatorMatch(Evaluator):
+    """
+    EvalatorMatch is a subclass of Evaluator specifically designed to check for direct matches
+    between target responses and reference responses of an instance. It assigns a binary score
+    based on whether any target response exactly matches a reference response.
+    """
+
+    def _evaluate(self, instance: Instance, **kwargs):
+        """
+        Evaluates the given instance by comparing each target response with reference responses.
+
+        If a target response matches any reference response, it is assigned a score of 1,
+        indicating a match. Otherwise, it receives a score of 0.
+
+        :param ~Instance instance: The instance to be evaluated, containing target and reference responses.
+        """
+        instance.eval_results = []  # Reset or initialize evaluation results
+
+        # Iterate over each target response in the instance
+        for response in instance.target_responses:
+            # Initialize the evaluation result for this response as 0 (no match)
+            eval_result = False
+
+            # Compare the current target response with each reference response
+            for reference in instance.reference_responses:
+                # If a match is found, set the evaluation result to 1
+                if response == reference:
+                    eval_result = True
+                    break  # No need to check further if a match is found
+
+            # Append the evaluation result (0 or 1) for this target response
+            instance.eval_results.append(eval_result)