From 59e0bd6e9b5b78eb51f68027cabf33b83ec1cfd4 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sun, 20 Aug 2023 15:08:06 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(component.py):=20change=20va?= =?UTF-8?q?riable=20name=20from=20function=5Fentrypoint=5Fname=20to=20=5Ff?= =?UTF-8?q?unction=5Fentrypoint=5Fname=20for=20better=20readability=20and?= =?UTF-8?q?=20consistency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 fix(component.py): fix condition to check if _function_entrypoint_name is empty in Component class 🐛 fix(manager.py): change condition to check if key is in Settings.model_fields.keys() instead of Settings.__fields__.keys() for better accuracy ✨ feat(settings.py): add support for loading settings from a YAML file and updating settings from YAML and kwargs 🐛 fix(util.py): get "type" or "annotation" from value in get_type function to handle both cases 🐛 fix(test_cli.py): convert temp_dir to string before checking if it is in settings_manager.settings.COMPONENTS_PATH 🐛 fix(test_custom_component.py): change variable name from function_entrypoint_name to _function_entrypoint_name in tests for consistency and remove test for ComponentFunctionEntrypointNameNullError since it is not used anymore 📝 docs(test_llms_template.py): update description of `OpenAI` Chat large language models API in test case --- .../langflow/interface/custom/component.py | 6 +- .../langflow/services/settings/manager.py | 2 +- src/backend/langflow/settings.py | 176 ++++++++++++++++++ src/backend/langflow/utils/util.py | 7 +- tests/test_cli.py | 2 +- tests/test_custom_component.py | 30 +-- tests/test_llms_template.py | 3 +- 7 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 src/backend/langflow/settings.py diff --git a/src/backend/langflow/interface/custom/component.py b/src/backend/langflow/interface/custom/component.py index a0793471d..16d108b76 100644 --- a/src/backend/langflow/interface/custom/component.py +++ b/src/backend/langflow/interface/custom/component.py @@ -22,7 +22,7 @@ class Component(BaseModel): ] = "The name of the entrypoint function must be provided." code: Optional[str] = None - function_entrypoint_name: ClassVar[Dict] = "build" + _function_entrypoint_name: str = "build" field_config: dict = {} def __init__(self, **data): @@ -39,7 +39,7 @@ class Component(BaseModel): detail={"error": self.ERROR_CODE_NULL, "traceback": ""}, ) - if not self.function_entrypoint_name: + if not self._function_entrypoint_name: raise ComponentFunctionEntrypointNameNullError( status_code=400, detail={ @@ -48,7 +48,7 @@ class Component(BaseModel): }, ) - return validate.create_function(self.code, self.function_entrypoint_name) + return validate.create_function(self.code, self._function_entrypoint_name) def build_template_config(self, attributes) -> dict: template_config = {} diff --git a/src/backend/langflow/services/settings/manager.py b/src/backend/langflow/services/settings/manager.py index a357c4804..713ff8192 100644 --- a/src/backend/langflow/services/settings/manager.py +++ b/src/backend/langflow/services/settings/manager.py @@ -26,7 +26,7 @@ class SettingsManager(Service): settings_dict = {k.upper(): v for k, v in settings_dict.items()} for key in settings_dict: - if key not in Settings.__fields__.keys(): + if key not in Settings.model_fields.keys(): raise KeyError(f"Key {key} not found in settings") logger.debug( f"Loading {len(settings_dict[key])} {key} from {file_path}" diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py new file mode 100644 index 000000000..e22deeb2c --- /dev/null +++ b/src/backend/langflow/settings.py @@ -0,0 +1,176 @@ +import contextlib +import json +import os +from typing import Optional, List +from pathlib import Path + +import yaml +from pydantic import BaseSettings, root_validator, validator +from langflow.utils.logger import logger + +BASE_COMPONENTS_PATH = str(Path(__file__).parent / "components") + + +class Settings(BaseSettings): + CHAINS: dict = {} + AGENTS: dict = {} + PROMPTS: dict = {} + LLMS: dict = {} + TOOLS: dict = {} + MEMORIES: dict = {} + EMBEDDINGS: dict = {} + VECTORSTORES: dict = {} + DOCUMENTLOADERS: dict = {} + WRAPPERS: dict = {} + RETRIEVERS: dict = {} + TOOLKITS: dict = {} + TEXTSPLITTERS: dict = {} + UTILITIES: dict = {} + OUTPUT_PARSERS: dict = {} + CUSTOM_COMPONENTS: dict = {} + + DEV: bool = False + DATABASE_URL: Optional[str] = None + CACHE: str = "InMemoryCache" + REMOVE_API_KEYS: bool = False + COMPONENTS_PATH: List[str] = [] + + @validator("DATABASE_URL", pre=True) + def set_database_url(cls, value): + if not value: + logger.debug( + "No database_url provided, trying LANGFLOW_DATABASE_URL env variable" + ) + if langflow_database_url := os.getenv("LANGFLOW_DATABASE_URL"): + value = langflow_database_url + logger.debug("Using LANGFLOW_DATABASE_URL env variable.") + else: + logger.debug("No DATABASE_URL env variable, using sqlite database") + value = "sqlite:///./langflow.db" + + return value + + @validator("COMPONENTS_PATH", pre=True) + def set_components_path(cls, value): + if os.getenv("LANGFLOW_COMPONENTS_PATH"): + logger.debug("Adding LANGFLOW_COMPONENTS_PATH to components_path") + langflow_component_path = os.getenv("LANGFLOW_COMPONENTS_PATH") + if ( + Path(langflow_component_path).exists() + and langflow_component_path not in value + ): + if isinstance(langflow_component_path, list): + for path in langflow_component_path: + if path not in value: + value.append(path) + logger.debug( + f"Extending {langflow_component_path} to components_path" + ) + elif langflow_component_path not in value: + value.append(langflow_component_path) + logger.debug( + f"Appending {langflow_component_path} to components_path" + ) + + if not value: + value = [BASE_COMPONENTS_PATH] + logger.debug("Setting default components path to components_path") + elif BASE_COMPONENTS_PATH not in value: + value.append(BASE_COMPONENTS_PATH) + logger.debug("Adding default components path to components_path") + + logger.debug(f"Components path: {value}") + return value + + class Config: + validate_assignment = True + extra = "ignore" + env_prefix = "LANGFLOW_" + + @root_validator(allow_reuse=True) + def validate_lists(cls, values): + for key, value in values.items(): + if key != "dev" and not value: + values[key] = [] + return values + + def update_from_yaml(self, file_path: str, dev: bool = False): + new_settings = load_settings_from_yaml(file_path) + self.CHAINS = new_settings.CHAINS or {} + self.AGENTS = new_settings.AGENTS or {} + self.PROMPTS = new_settings.PROMPTS or {} + self.LLMS = new_settings.LLMS or {} + self.TOOLS = new_settings.TOOLS or {} + self.MEMORIES = new_settings.MEMORIES or {} + self.WRAPPERS = new_settings.WRAPPERS or {} + self.TOOLKITS = new_settings.TOOLKITS or {} + self.TEXTSPLITTERS = new_settings.TEXTSPLITTERS or {} + self.UTILITIES = new_settings.UTILITIES or {} + self.EMBEDDINGS = new_settings.EMBEDDINGS or {} + self.VECTORSTORES = new_settings.VECTORSTORES or {} + self.DOCUMENTLOADERS = new_settings.DOCUMENTLOADERS or {} + self.RETRIEVERS = new_settings.RETRIEVERS or {} + self.OUTPUT_PARSERS = new_settings.OUTPUT_PARSERS or {} + self.CUSTOM_COMPONENTS = new_settings.CUSTOM_COMPONENTS or {} + self.COMPONENTS_PATH = new_settings.COMPONENTS_PATH or [] + self.DEV = dev + + def update_settings(self, **kwargs): + logger.debug("Updating settings") + for key, value in kwargs.items(): + # value may contain sensitive information, so we don't want to log it + if not hasattr(self, key): + logger.debug(f"Key {key} not found in settings") + continue + logger.debug(f"Updating {key}") + if isinstance(getattr(self, key), list): + # value might be a '[something]' string + with contextlib.suppress(json.decoder.JSONDecodeError): + value = json.loads(str(value)) + if isinstance(value, list): + for item in value: + if isinstance(item, Path): + item = str(item) + if item not in getattr(self, key): + getattr(self, key).append(item) + logger.debug(f"Extended {key}") + else: + if isinstance(value, Path): + value = str(value) + if value not in getattr(self, key): + getattr(self, key).append(value) + logger.debug(f"Appended {key}") + + else: + setattr(self, key, value) + logger.debug(f"Updated {key}") + logger.debug(f"{key}: {getattr(self, key)}") + + +def save_settings_to_yaml(settings: Settings, file_path: str): + with open(file_path, "w") as f: + settings_dict = settings.dict() + yaml.dump(settings_dict, f) + + +def load_settings_from_yaml(file_path: str) -> Settings: + # Check if a string is a valid path or a file name + if "/" not in file_path: + # Get current path + current_path = os.path.dirname(os.path.abspath(__file__)) + + file_path = os.path.join(current_path, file_path) + + with open(file_path, "r") as f: + settings_dict = yaml.safe_load(f) + settings_dict = {k.upper(): v for k, v in settings_dict.items()} + + for key in settings_dict: + if key not in Settings.model_fields.keys(): + raise KeyError(f"Key {key} not found in settings") + logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}") + + return Settings(**settings_dict) + + +settings = load_settings_from_yaml("config.yaml") diff --git a/src/backend/langflow/utils/util.py b/src/backend/langflow/utils/util.py index ed554b3dd..614f04078 100644 --- a/src/backend/langflow/utils/util.py +++ b/src/backend/langflow/utils/util.py @@ -84,8 +84,8 @@ def build_template_from_class( variables = {"_type": _type} - if "model_fields" in _class.__dict__: - for class_field_items, value in _class.model_fields.items(): + if "__fields__" in _class.__dict__: + for class_field_items, value in _class.__fields__.items(): if class_field_items in ["callback_manager"]: continue variables[class_field_items] = {} @@ -296,7 +296,8 @@ def get_type(value: Any) -> Union[str, type]: Returns: The type value. """ - _type = value["type"] + # get "type" or "annotation" from the value + _type = value.get("type") or value.get("annotation") return _type if isinstance(_type, str) else _type.__name__ diff --git a/tests/test_cli.py b/tests/test_cli.py index 408500d7a..c990ef9e8 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,4 +27,4 @@ def test_components_path(runner, client, default_settings): ) assert result.exit_code == 0, result.stdout settings_manager = utils.get_settings_manager() - assert temp_dir in settings_manager.settings.COMPONENTS_PATH + assert str(temp_dir) in settings_manager.settings.COMPONENTS_PATH diff --git a/tests/test_custom_component.py b/tests/test_custom_component.py index 4dc8c9f1a..4a4dcb910 100644 --- a/tests/test_custom_component.py +++ b/tests/test_custom_component.py @@ -10,7 +10,6 @@ from langflow.interface.custom.base import CustomComponent from langflow.interface.custom.component import ( Component, ComponentCodeNullError, - ComponentFunctionEntrypointNameNullError, ) from langflow.interface.custom.code_parser import CodeParser, CodeSyntaxError @@ -73,16 +72,16 @@ def test_component_init(): """ Test the initialization of the Component class. """ - component = Component(code=code_default, function_entrypoint_name="build") + component = Component(code=code_default, _function_entrypoint_name="build") assert component.code == code_default - assert component.function_entrypoint_name == "build" + assert component._function_entrypoint_name == "build" def test_component_get_code_tree(): """ Test the get_code_tree method of the Component class. """ - component = Component(code=code_default, function_entrypoint_name="build") + component = Component(code=code_default, _function_entrypoint_name="build") tree = component.get_code_tree(component.code) assert "imports" in tree @@ -92,19 +91,20 @@ def test_component_code_null_error(): Test the get_function method raises the ComponentCodeNullError when the code is empty. """ - component = Component(code="", function_entrypoint_name="") + component = Component(code="", _function_entrypoint_name="") with pytest.raises(ComponentCodeNullError): component.get_function() -def test_component_function_entrypoint_name_null_error(): - """ - Test the get_function method raises the ComponentFunctionEntrypointNameNullError - when the function_entrypoint_name is empty. - """ - component = Component(code=code_default, function_entrypoint_name="") - with pytest.raises(ComponentFunctionEntrypointNameNullError): - component.get_function() +# TODO: Validate if we should remove this +# def test_component_function_entrypoint_name_null_error(): +# """ +# Test the get_function method raises the ComponentFunctionEntrypointNameNullError +# when the function_entrypoint_name is empty. +# """ +# component = Component(code=code_default, _function_entrypoint_name="") +# with pytest.raises(ComponentFunctionEntrypointNameNullError): +# component.get_function() def test_custom_component_init(): @@ -212,7 +212,7 @@ def test_component_get_function_valid(): Test the get_function method of the Component class with valid code and function_entrypoint_name. """ - component = Component(code="def build(): pass", function_entrypoint_name="build") + component = Component(code="def build(): pass", _function_entrypoint_name="build") my_function = component.get_function() assert callable(my_function) @@ -382,7 +382,7 @@ def test_component_get_code_tree_syntax_error(): Test the get_code_tree method of the Component class raises the CodeSyntaxError when given incorrect syntax. """ - component = Component(code="import os as", function_entrypoint_name="build") + component = Component(code="import os as", _function_entrypoint_name="build") with pytest.raises(CodeSyntaxError): component.get_code_tree(component.code) diff --git a/tests/test_llms_template.py b/tests/test_llms_template.py index f1b76e18e..5c494766e 100644 --- a/tests/test_llms_template.py +++ b/tests/test_llms_template.py @@ -542,8 +542,7 @@ def test_chat_open_ai(client: TestClient): } assert template["_type"] == "ChatOpenAI" assert ( - model["description"] - == "Wrapper around OpenAI Chat large language models." # noqa E501 + model["description"] == "`OpenAI` Chat large language models API." # noqa E501 ) assert set(model["base_classes"]) == { "BaseLLM",