From 81da52437523673cf553f9bd12e3792eca1af667 Mon Sep 17 00:00:00 2001 From: VICTOR CORREA GOMES <112295415+Vigtu@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:26:10 -0300 Subject: [PATCH] feat(components): add LangWatch evaluator component - New Bundle (#4722) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(components): add LangWatch evaluator component * feat(langwatch): add tracing integration and custom endpoint support * style(langwatch): update component name and svg icon * [autofix.ci] apply automated fixes * Clean code with code formatting styles * Add contexts and expected_output also as dynamic fields * refactor(langwatch): remove redundant logging and improve type hinting - Removed unnecessary logger exception calls in error handling sections to streamline the code. - Added type hinting for the trace_id assignment to enhance code clarity and maintainability. --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Rogério Chaves Co-authored-by: Gabriel Luiz Freitas Almeida --- .../langflow/components/langwatch/__init__.py | 3 + .../components/langwatch/langwatch.py | 292 +++ src/frontend/src/constants/constants.ts | 2 + .../src/icons/Langwatch/langwatch-icon.svg | 2215 +---------------- .../src/icons/Langwatch/langwatch.jsx | 7 +- src/frontend/src/utils/styleUtils.ts | 1 + 6 files changed, 305 insertions(+), 2215 deletions(-) create mode 100644 src/backend/base/langflow/components/langwatch/__init__.py create mode 100644 src/backend/base/langflow/components/langwatch/langwatch.py diff --git a/src/backend/base/langflow/components/langwatch/__init__.py b/src/backend/base/langflow/components/langwatch/__init__.py new file mode 100644 index 000000000..dc3149bfb --- /dev/null +++ b/src/backend/base/langflow/components/langwatch/__init__.py @@ -0,0 +1,3 @@ +from .langwatch import LangWatchComponent + +__all__ = ["LangWatchComponent"] diff --git a/src/backend/base/langflow/components/langwatch/langwatch.py b/src/backend/base/langflow/components/langwatch/langwatch.py new file mode 100644 index 000000000..b49517696 --- /dev/null +++ b/src/backend/base/langflow/components/langwatch/langwatch.py @@ -0,0 +1,292 @@ +import json +import logging +import os +from typing import Any + +import httpx + +from langflow.custom import Component +from langflow.inputs.inputs import MultilineInput +from langflow.io import ( + BoolInput, + DropdownInput, + FloatInput, + IntInput, + MessageTextInput, + NestedDictInput, + Output, + SecretStrInput, +) +from langflow.schema import Data +from langflow.schema.dotdict import dotdict + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +class LangWatchComponent(Component): + display_name: str = "LangWatch Evaluator" + description: str = "Evaluates various aspects of language models using LangWatch's evaluation endpoints." + documentation: str = "https://docs.langwatch.ai/langevals/documentation/introduction" + icon: str = "Langwatch" + name: str = "LangWatchEvaluator" + + inputs = [ + DropdownInput( + name="evaluator_name", + display_name="Evaluator Name", + options=[], + required=True, + info="Select an evaluator.", + refresh_button=True, + real_time_refresh=True, + ), + SecretStrInput( + name="api_key", + display_name="API Key", + required=True, + info="Enter your LangWatch API key.", + ), + MessageTextInput( + name="input", + display_name="Input", + required=False, + info="The input text for evaluation.", + ), + MessageTextInput( + name="output", + display_name="Output", + required=False, + info="The output text for evaluation.", + ), + MessageTextInput( + name="expected_output", + display_name="Expected Output", + required=False, + info="The expected output for evaluation.", + ), + MessageTextInput( + name="contexts", + display_name="Contexts", + required=False, + info="The contexts for evaluation (comma-separated).", + ), + IntInput( + name="timeout", + display_name="Timeout", + info="The maximum time (in seconds) allowed for the server to respond before timing out.", + value=30, + advanced=True, + ), + ] + + outputs = [ + Output(name="evaluation_result", display_name="Evaluation Result", method="evaluate"), + ] + + def __init__(self, **data): + super().__init__(**data) + self.evaluators = self.get_evaluators() + self.dynamic_inputs = {} + self._code = data.get("_code", "") + self.current_evaluator = None + if self.evaluators: + self.current_evaluator = next(iter(self.evaluators)) + + def get_evaluators(self) -> dict[str, Any]: + url = f"{os.getenv('LANGWATCH_ENDPOINT', 'https://app.langwatch.ai')}/api/evaluations/list" + try: + response = httpx.get(url, timeout=10) + response.raise_for_status() + data = response.json() + return data.get("evaluators", {}) + except httpx.RequestError as e: + self.status = f"Error fetching evaluators: {e}" + return {} + + def update_build_config(self, build_config: dotdict, field_value: Any, field_name: str | None = None) -> dotdict: + try: + logger.info("Updating build config. Field name: %s, Field value: %s", field_name, field_value) + + if field_name is None or field_name == "evaluator_name": + self.evaluators = self.get_evaluators() + build_config["evaluator_name"]["options"] = list(self.evaluators.keys()) + + # Set a default evaluator if none is selected + if not self.current_evaluator and self.evaluators: + self.current_evaluator = next(iter(self.evaluators)) + build_config["evaluator_name"]["value"] = self.current_evaluator + + # Define default keys that should always be present + default_keys = ["code", "_type", "evaluator_name", "api_key", "input", "output", "timeout"] + + if field_value and field_value in self.evaluators and self.current_evaluator != field_value: + self.current_evaluator = field_value + evaluator = self.evaluators[field_value] + + # Clear previous dynamic inputs + keys_to_remove = [key for key in build_config if key not in default_keys] + for key in keys_to_remove: + del build_config[key] + + # Clear component's dynamic attributes + for attr in list(self.__dict__.keys()): + if attr not in default_keys and attr not in [ + "evaluators", + "dynamic_inputs", + "_code", + "current_evaluator", + ]: + delattr(self, attr) + + # Add new dynamic inputs + self.dynamic_inputs = self.get_dynamic_inputs(evaluator) + for name, input_config in self.dynamic_inputs.items(): + build_config[name] = input_config.to_dict() + + # Update required fields + required_fields = {"api_key", "evaluator_name"}.union(evaluator.get("requiredFields", [])) + for key in build_config: + if isinstance(build_config[key], dict): + build_config[key]["required"] = key in required_fields + + # Validate presence of default keys + missing_keys = [key for key in default_keys if key not in build_config] + if missing_keys: + logger.warning("Missing required keys in build_config: %s", missing_keys) + # Add missing keys with default values + for key in missing_keys: + build_config[key] = {"value": None, "type": "str"} + + # Ensure the current_evaluator is always set in the build_config + build_config["evaluator_name"]["value"] = self.current_evaluator + + logger.info("Current evaluator set to: %s", self.current_evaluator) + return build_config + + except (KeyError, AttributeError, ValueError) as e: + self.status = f"Error updating component: {e!s}" + return build_config + else: + return build_config + + def get_dynamic_inputs(self, evaluator: dict[str, Any]): + try: + dynamic_inputs = {} + + input_fields = [ + field + for field in evaluator.get("requiredFields", []) + evaluator.get("optionalFields", []) + if field not in ["input", "output"] + ] + + for field in input_fields: + input_params = { + "name": field, + "display_name": field.replace("_", " ").title(), + "required": field in evaluator.get("requiredFields", []), + } + if field == "contexts": + dynamic_inputs[field] = MultilineInput(**input_params, multiline=True) + else: + dynamic_inputs[field] = MessageTextInput(**input_params) + + settings = evaluator.get("settings", {}) + for setting_name, setting_config in settings.items(): + schema = evaluator.get("settings_json_schema", {}).get("properties", {}).get(setting_name, {}) + + input_params = { + "name": setting_name, + "display_name": setting_name.replace("_", " ").title(), + "info": setting_config.get("description", ""), + "required": False, + } + + if schema.get("type") == "object": + input_type = NestedDictInput + input_params["value"] = schema.get("default", setting_config.get("default", {})) + elif schema.get("type") == "boolean": + input_type = BoolInput + input_params["value"] = schema.get("default", setting_config.get("default", False)) + elif schema.get("type") == "number": + is_float = isinstance(schema.get("default", setting_config.get("default")), float) + input_type = FloatInput if is_float else IntInput + input_params["value"] = schema.get("default", setting_config.get("default", 0)) + elif "enum" in schema: + input_type = DropdownInput + input_params["options"] = schema["enum"] + input_params["value"] = schema.get("default", setting_config.get("default")) + else: + input_type = MessageTextInput + default_value = schema.get("default", setting_config.get("default")) + input_params["value"] = str(default_value) if default_value is not None else "" + + dynamic_inputs[setting_name] = input_type(**input_params) + + except (KeyError, AttributeError, ValueError, TypeError) as e: + self.status = f"Error creating dynamic inputs: {e!s}" + return {} + return dynamic_inputs + + async def evaluate(self) -> Data: + if not self.api_key: + return Data(data={"error": "API key is required"}) + + # Prioritize evaluator_name if it exists + evaluator_name = getattr(self, "evaluator_name", None) or self.current_evaluator + + if not evaluator_name: + if self.evaluators: + evaluator_name = next(iter(self.evaluators)) + logger.info("No evaluator was selected. Using default: %s", evaluator_name) + else: + return Data( + data={"error": "No evaluator selected and no evaluators available. Please choose an evaluator."} + ) + + try: + evaluator = self.evaluators.get(evaluator_name) + if not evaluator: + return Data(data={"error": f"Selected evaluator '{evaluator_name}' not found."}) + + logger.info("Evaluating with evaluator: %s", evaluator_name) + + endpoint = f"/api/evaluations/{evaluator_name}/evaluate" + url = f"{os.getenv('LANGWATCH_ENDPOINT', 'https://app.langwatch.ai')}{endpoint}" + + headers = {"Content-Type": "application/json", "X-Auth-Token": self.api_key} + + payload = { + "data": { + "input": self.input, + "output": self.output, + "expected_output": self.expected_output, + "contexts": self.contexts.split(",") if self.contexts else [], + }, + "settings": {}, + } + + if ( + self._tracing_service + and self._tracing_service._tracers + and "langwatch" in self._tracing_service._tracers + ): + payload["trace_id"] = str(self._tracing_service._tracers["langwatch"].trace_id) # type: ignore[assignment] + + for setting_name in self.dynamic_inputs: + payload["settings"][setting_name] = getattr(self, setting_name, None) + + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post(url, json=payload, headers=headers) + + response.raise_for_status() + result = response.json() + + formatted_result = json.dumps(result, indent=2) + self.status = f"Evaluation completed successfully. Result:\n{formatted_result}" + return Data(data=result) + + except (httpx.RequestError, KeyError, AttributeError, ValueError) as e: + error_message = f"Evaluation error: {e!s}" + self.status = error_message + return Data(data={"error": error_message}) diff --git a/src/frontend/src/constants/constants.ts b/src/frontend/src/constants/constants.ts index 176b148d8..a33136f22 100644 --- a/src/frontend/src/constants/constants.ts +++ b/src/frontend/src/constants/constants.ts @@ -740,6 +740,8 @@ export const BUNDLES_SIDEBAR_FOLDER_NAMES = [ "Notion", "AssemblyAI", "assemblyai", + "LangWatch", + "langwatch", ]; export const AUTHORIZED_DUPLICATE_REQUESTS = [ diff --git a/src/frontend/src/icons/Langwatch/langwatch-icon.svg b/src/frontend/src/icons/Langwatch/langwatch-icon.svg index 917ba30bd..d8d8a70bf 100644 --- a/src/frontend/src/icons/Langwatch/langwatch-icon.svg +++ b/src/frontend/src/icons/Langwatch/langwatch-icon.svg @@ -1,2213 +1,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + \ No newline at end of file diff --git a/src/frontend/src/icons/Langwatch/langwatch.jsx b/src/frontend/src/icons/Langwatch/langwatch.jsx index 613b96a2f..1ac26f439 100644 --- a/src/frontend/src/icons/Langwatch/langwatch.jsx +++ b/src/frontend/src/icons/Langwatch/langwatch.jsx @@ -1,9 +1,10 @@ const SvgLangwatch = (props) => (