From 548383b09e7c67e61738f0a48623ce3be9d53592 Mon Sep 17 00:00:00 2001 From: gustavoschaedler Date: Fri, 21 Jul 2023 19:13:52 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(endpoints.py):=20add=20missi?= =?UTF-8?q?ng=20import=20statement=20for=20'settings'=20module=20=E2=9C=A8?= =?UTF-8?q?=20feat(endpoints.py):=20add=20support=20for=20loading=20custom?= =?UTF-8?q?=20components=20from=20a=20specified=20path=20=F0=9F=90=9B=20fi?= =?UTF-8?q?x(endpoints.py):=20fix=20typo=20in=20variable=20name=20'custom?= =?UTF-8?q?=5Fcomponents=5Ffrom=5Ffile'=20=F0=9F=90=9B=20fix(endpoints.py)?= =?UTF-8?q?:=20fix=20typo=20in=20variable=20name=20'filtered'=20?= =?UTF-8?q?=F0=9F=90=9B=20fix(base.py):=20fix=20indentation=20of=20raise?= =?UTF-8?q?=20statement=20to=20improve=20code=20readability=20=F0=9F=90=9B?= =?UTF-8?q?=20fix(component.py):=20fix=20indentation=20of=20if=20statement?= =?UTF-8?q?s=20to=20improve=20code=20readability=20=F0=9F=90=9B=20fix(load?= =?UTF-8?q?=5Fcustom=5Fcomponent=5Ffrom=5Fpath.py):=20fix=20indentation=20?= =?UTF-8?q?of=20base=5Fpath=20comment=20to=20improve=20code=20readability?= =?UTF-8?q?=20=F0=9F=90=9B=20fix(load=5Fcustom=5Fcomponent=5Ffrom=5Fpath.p?= =?UTF-8?q?y):=20fix=20indentation=20of=20base=5Fpath=20assignment=20to=20?= =?UTF-8?q?improve=20code=20readability=20=F0=9F=90=9B=20fix(load=5Fcustom?= =?UTF-8?q?=5Fcomponent=5Ffrom=5Fpath.py):=20fix=20indentation=20of=20vali?= =?UTF-8?q?date=5Fcode=20method=20to=20improve=20code=20readability=20?= =?UTF-8?q?=F0=9F=90=9B=20fix(load=5Fcustom=5Fcomponent=5Ffrom=5Fpath.py):?= =?UTF-8?q?=20fix=20indentation=20of=20build=5Fcomponent=5Fmenu=5Flist=20m?= =?UTF-8?q?ethod=20to=20improve=20code=20readability=20=F0=9F=90=9B=20fix(?= =?UTF-8?q?types.py):=20fix=20indentation=20of=20extract=5Ftype=5Ffrom=5Fo?= =?UTF-8?q?ptional=20function=20to=20improve=20code=20readability=20?= =?UTF-8?q?=E2=9C=A8=20feat(types.py):=20add=20support=20for=20building=20?= =?UTF-8?q?custom=20component=20templates=20=E2=9C=A8=20feat(types.py):=20?= =?UTF-8?q?add=20support=20for=20building=20custom=20component=20templates?= =?UTF-8?q?=20with=20extra=20fields=20=E2=9C=A8=20feat(types.py):=20add=20?= =?UTF-8?q?support=20for=20building=20custom=20component=20templates=20wit?= =?UTF-8?q?h=20function=20arguments=20=E2=9C=A8=20feat(types.py):=20add=20?= =?UTF-8?q?support=20for=20building=20custom=20component=20templates=20wit?= =?UTF-8?q?h=20base=20classes=20=E2=9C=A8=20feat(types.py):=20add=20suppor?= =?UTF-8?q?t=20for=20building=20custom=20component=20templates=20with=20re?= =?UTF-8?q?turn=20type=20=E2=9C=A8=20feat(types.py):=20add=20support=20for?= =?UTF-8?q?=20building=20custom=20component=20templates=20with=20exception?= =?UTF-8?q?=20handling=20=E2=9C=A8=20feat(types.py):=20add=20support=20for?= =?UTF-8?q?=20building=20custom=20component=20templates=20from=20a=20speci?= =?UTF-8?q?fied=20path=20=F0=9F=90=9B=20fix(settings.py):=20fix=20indentat?= =?UTF-8?q?ion=20of=20set=5Fenv=5Fvariables=20method=20to=20improve=20code?= =?UTF-8?q?=20readability=20=E2=9C=A8=20feat(settings.py):=20add=20support?= =?UTF-8?q?=20for=20specifying=20component=20path=20in=20settings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 fix(test_custom_component.py): change variable names and attributes in YourComponent class for better readability and consistency 🐛 fix(test_custom_component.py): reformat code to adhere to PEP8 style guide 🐛 fix(test_custom_component.py): fix syntax error in test_component_get_code_tree_syntax_error() 🐛 fix(test_custom_component.py): fix syntax error in test_custom_component_class_template_validation_no_code() 🐛 fix(test_custom_component.py): fix syntax error in test_custom_component_get_main_class_name_no_main_class() ✨ feat(test_custom_component.py): add test_component_get_function_valid() to test the get_function method of the Component class with valid code and function_entrypoint_name ✨ feat(test_custom_component.py): add test_code_parser_parse_assign() to test the parse_assign method of the CodeParser class ✨ feat(test_custom_component.py): add test_custom_component_class_template_validation_no_code() to test the _class_template_validation method of the CustomComponent class when the code is None ✨ feat(test_custom_component.py): add test_custom_component_get_main_class_name_no_main_class() to test the get_main_class_name method of the CustomComponent class when there is no main class ✨ feat(test_custom_component.py): add test_custom_component_get_function_entrypoint_args_no_args() to test the get_function_entrypoint_args method of the CustomComponent class when there are no arguments ✨ feat(test_custom_component.py): add test_custom_component_get_function_entrypoint_return_type_none() to test the get_function_entrypoint_return_type method of the CustomComponent class when the return type is None --- src/backend/langflow/api/v1/endpoints.py | 15 ++++++++++-- .../langflow/interface/custom/component.py | 6 ++--- .../langflow/interface/custom/constants.py | 6 ++--- .../custom/load_custom_component_from_path.py | 21 +++++++++++++++- src/backend/langflow/interface/types.py | 24 +++++++++++++++---- src/backend/langflow/settings.py | 13 ++++++++-- tests/test_custom_component.py | 6 ++--- 7 files changed, 73 insertions(+), 18 deletions(-) diff --git a/src/backend/langflow/api/v1/endpoints.py b/src/backend/langflow/api/v1/endpoints.py index b5dc8dfa9..79eb90ac3 100644 --- a/src/backend/langflow/api/v1/endpoints.py +++ b/src/backend/langflow/api/v1/endpoints.py @@ -5,6 +5,7 @@ from langflow.cache.utils import save_uploaded_file from langflow.database.models.flow import Flow from langflow.processing.process import process_graph_cached, process_tweaks from langflow.utils.logger import logger +from langflow.settings import settings from fastapi import APIRouter, Depends, HTTPException, UploadFile @@ -35,7 +36,16 @@ router = APIRouter(tags=["Base"]) @router.get("/all") def get_all(): - return build_langchain_types_dict() + native_components = build_langchain_types_dict() + + if settings.component_path: + custom_components_from_file = build_langchain_custom_component_list_from_path( + str(settings.component_path[0]) + ) + else: + custom_components_from_file = {} + + return {**native_components, **custom_components_from_file} @router.get("/load_custom_component_from_path") @@ -59,8 +69,9 @@ def get_load_custom_component_from_path_test(path: str): reader = DirectoryReader(path, False) file_list = reader.get_files() + data = reader.build_component_menu_list(file_list) - return reader.build_component_menu_list(file_list) + return reader.filter_loaded_components(data, True) # For backwards compatibility we will keep the old endpoint diff --git a/src/backend/langflow/interface/custom/component.py b/src/backend/langflow/interface/custom/component.py index 5e84c235e..3db2a2516 100644 --- a/src/backend/langflow/interface/custom/component.py +++ b/src/backend/langflow/interface/custom/component.py @@ -56,13 +56,13 @@ class Component(BaseModel): item_name = item.get("name") if item_value := item.get("value"): - if "langflow_display_name" in item_name: + if "display_name" in item_name: template_config["display_name"] = ast.literal_eval(item_value) - elif "langflow_description" in item_name: + elif "description" in item_name: template_config["description"] = ast.literal_eval(item_value) - elif "langflow_field_config" in item_name: + elif "field_config" in item_name: template_config["field_config"] = ast.literal_eval(item_value) return template_config diff --git a/src/backend/langflow/interface/custom/constants.py b/src/backend/langflow/interface/custom/constants.py index 6bf4f4651..a001e9091 100644 --- a/src/backend/langflow/interface/custom/constants.py +++ b/src/backend/langflow/interface/custom/constants.py @@ -35,9 +35,9 @@ from langchain.schema import Document import requests class YourComponent(CustomComponent): - langflow_display_name: str = "Your Component" - langflow_description: str = "Your description" - langflow_field_config = { "url": { "multiline": True, "required": True } } + display_name: str = "Your Component" + description: str = "Your description" + field_config = { "url": { "multiline": True, "required": True } } def build(self, url: str, llm: BaseLLM, template: Prompt) -> Document: response = requests.get(url) diff --git a/src/backend/langflow/interface/custom/load_custom_component_from_path.py b/src/backend/langflow/interface/custom/load_custom_component_from_path.py index 16c846f48..6842dbee4 100644 --- a/src/backend/langflow/interface/custom/load_custom_component_from_path.py +++ b/src/backend/langflow/interface/custom/load_custom_component_from_path.py @@ -34,7 +34,9 @@ class StringCompressor: class DirectoryReader: - base_path = "/custom_component_files" + # Ensure the base path to read the files that contain + # the custom components from this directory. + base_path = "" def __init__(self, directory_path, compress_code_field=False): """ @@ -59,6 +61,22 @@ class DirectoryReader: """ return len(file_content.strip()) == 0 + def filter_loaded_components(self, data: dict, with_errors: bool) -> dict: + items = [ + { + "name": menu["name"], + "path": menu["path"], + "components": [ + component + for component in menu["components"] + if (component["error"] if with_errors else not component["error"]) + ], + } + for menu in data["menu"] + ] + filtred = [menu for menu in items if menu["components"]] + return {"menu": filtred} + def validate_code(self, file_content): """ Validate the Python code by trying to parse it with ast.parse. @@ -92,6 +110,7 @@ class DirectoryReader: raise CustomComponentPathValueError( f"The path needs to start with '{self.base_path}'." ) + file_list = [] for root, _, files in os.walk(safe_path): file_list.extend( diff --git a/src/backend/langflow/interface/types.py b/src/backend/langflow/interface/types.py index b33e2a397..ba645bb15 100644 --- a/src/backend/langflow/interface/types.py +++ b/src/backend/langflow/interface/types.py @@ -244,10 +244,26 @@ def build_langchain_custom_component_list_from_path(path: str): # Build and validate all files data = reader.build_component_menu_list(file_list) + valid_components = reader.filter_loaded_components(data, False) - raw_code = data.get("menu")[0].get("components")[0].get("code") + # TODO: Handle those invalid components + reader.filter_loaded_components(data, True) - extractor = CustomComponent(code=raw_code) - extractor.is_check_valid() + menu = {} + for menu_item in valid_components["menu"]: + menu_name = menu_item["name"] + menu[menu_name] = {} - return build_langchain_template_custom_component(extractor) + for component in menu_item["components"]: + component_name = component["name"] + component_code = component["code"] + + component_extractor = CustomComponent(code=component_code) + component_extractor.is_check_valid() + component_template = build_langchain_template_custom_component( + component_extractor + ) + + menu[menu_name][component_name] = component_template + + return menu diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py index 5966d2711..06f925a3c 100644 --- a/src/backend/langflow/settings.py +++ b/src/backend/langflow/settings.py @@ -1,5 +1,6 @@ import os -from typing import Optional +from typing import Optional, List +from pathlib import Path import yaml from pydantic import BaseSettings, root_validator @@ -28,9 +29,10 @@ class Settings(BaseSettings): database_url: Optional[str] = None cache: str = "InMemoryCache" remove_api_keys: bool = False + component_path: List[Path] @root_validator(pre=True) - def set_database_url(cls, values): + def set_env_variables(cls, values): if "database_url" not in values: logger.debug( "No database_url provided, trying LANGFLOW_DATABASE_URL env variable" @@ -40,6 +42,12 @@ class Settings(BaseSettings): else: logger.debug("No DATABASE_URL env variable, using sqlite database") values["database_url"] = "sqlite:///./langflow.db" + + values["component_path"] = [Path(__file__).parent / "components"] + + if os.getenv("LANGFLOW_COMPONENT_PATH"): + values["component_path"].append(Path(os.getenv("LANGFLOW_COMPONENT_PATH"))) + return values class Config: @@ -71,6 +79,7 @@ class Settings(BaseSettings): self.retrievers = new_settings.retrievers or {} self.output_parsers = new_settings.output_parsers or {} self.custom_components = new_settings.custom_components or {} + self.component_path = new_settings.component_path or [] self.dev = dev def update_settings(self, **kwargs): diff --git a/tests/test_custom_component.py b/tests/test_custom_component.py index e60b392ea..f4e57d10d 100644 --- a/tests/test_custom_component.py +++ b/tests/test_custom_component.py @@ -24,9 +24,9 @@ from langchain.schema import Document import requests class YourComponent(CustomComponent): - langflow_display_name: str = "Your Component" - langflow_description: str = "Your description" - langflow_field_config = { "url": { "multiline": True, "required": True } } + display_name: str = "Your Component" + description: str = "Your description" + field_config = { "url": { "multiline": True, "required": True } } def build(self, url: str, llm: BaseLLM, template: Prompt) -> Document: response = requests.get(url)