From 73e78b6b682df434d4a8048cd5c02a885e8c96bc Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 22 Sep 2023 14:03:15 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=80=20merge(listing.py):=20merge=20cha?= =?UTF-8?q?nges=20from=20types.py=20to=20improve=20code=20organization=20a?= =?UTF-8?q?nd=20readability=20=F0=9F=94=80=20merge(types.py):=20merge=20ch?= =?UTF-8?q?anges=20from=20listing.py=20to=20consolidate=20all=20type=20dic?= =?UTF-8?q?tionaries=20into=20a=20single=20function=20for=20better=20maint?= =?UTF-8?q?ainability=20and=20readability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/interface/listing.py | 39 ++----------- src/backend/langflow/interface/types.py | 68 ++++++++++++++++++++++- 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/backend/langflow/interface/listing.py b/src/backend/langflow/interface/listing.py index 1cab1efbc..2851e4ea8 100644 --- a/src/backend/langflow/interface/listing.py +++ b/src/backend/langflow/interface/listing.py @@ -1,19 +1,4 @@ -from langflow.interface.agents.base import agent_creator -from langflow.interface.chains.base import chain_creator -from langflow.interface.document_loaders.base import documentloader_creator -from langflow.interface.embeddings.base import embedding_creator -from langflow.interface.llms.base import llm_creator -from langflow.interface.memories.base import memory_creator -from langflow.interface.prompts.base import prompt_creator -from langflow.interface.text_splitters.base import textsplitter_creator -from langflow.interface.toolkits.base import toolkits_creator -from langflow.interface.tools.base import tool_creator -from langflow.interface.utilities.base import utility_creator -from langflow.interface.vector_store.base import vectorstore_creator -from langflow.interface.wrappers.base import wrapper_creator -from langflow.interface.output_parsers.base import output_parser_creator -from langflow.interface.retrievers.base import retriever_creator -from langflow.interface.custom.base import custom_component_creator +from langflow.services.utils import get_settings_service from langflow.utils.lazy_load import LazyLoadDictBase @@ -33,24 +18,10 @@ class AllTypesDict(LazyLoadDictBase): } def get_type_dict(self): - return { - "agents": agent_creator.to_list(), - "prompts": prompt_creator.to_list(), - "llms": llm_creator.to_list(), - "tools": tool_creator.to_list(), - "chains": chain_creator.to_list(), - "memory": memory_creator.to_list(), - "toolkits": toolkits_creator.to_list(), - "wrappers": wrapper_creator.to_list(), - "documentLoaders": documentloader_creator.to_list(), - "vectorStore": vectorstore_creator.to_list(), - "embeddings": embedding_creator.to_list(), - "textSplitters": textsplitter_creator.to_list(), - "utilities": utility_creator.to_list(), - "outputParsers": output_parser_creator.to_list(), - "retrievers": retriever_creator.to_list(), - "custom_components": custom_component_creator.to_list(), - } + from langflow.interface.types import get_all_types_dict + + settings_service = get_settings_service() + return get_all_types_dict(settings_service=settings_service) lazy_load_dict = AllTypesDict() diff --git a/src/backend/langflow/interface/types.py b/src/backend/langflow/interface/types.py index fa9967649..25d3e8c8b 100644 --- a/src/backend/langflow/interface/types.py +++ b/src/backend/langflow/interface/types.py @@ -1,7 +1,7 @@ import ast import contextlib from typing import Any, List -from langflow.api.utils import merge_nested_dicts_with_renaming +from langflow.api.utils import get_new_key from langflow.interface.agents.base import agent_creator from langflow.interface.chains.base import chain_creator from langflow.interface.custom.constants import CUSTOM_COMPONENT_SUPPORTED_TYPES @@ -433,6 +433,24 @@ def build_invalid_menu(invalid_components): return invalid_menu +def merge_nested_dicts_with_renaming(dict1, dict2): + for key, value in dict2.items(): + if ( + key in dict1 + and isinstance(value, dict) + and isinstance(dict1.get(key), dict) + ): + for sub_key, sub_value in value.items(): + if sub_key in dict1[key]: + new_key = get_new_key(dict1[key], sub_key) + dict1[key][new_key] = sub_value + else: + dict1[key][sub_key] = sub_value + else: + dict1[key] = value + return dict1 + + def build_langchain_custom_component_list_from_path(path: str): """Build a list of custom components for the langchain from a given path""" file_list = load_files_from_path(path) @@ -446,3 +464,51 @@ def build_langchain_custom_component_list_from_path(path: str): invalid_menu = build_invalid_menu(invalid_components) return merge_nested_dicts_with_renaming(valid_menu, invalid_menu) + + +def get_all_types_dict(settings_service): + native_components = build_langchain_types_dict() + # custom_components is a list of dicts + # need to merge all the keys into one dict + custom_components_from_file: dict[str, Any] = {} + if settings_service.settings.COMPONENTS_PATH: + logger.info( + f"Building custom components from {settings_service.settings.COMPONENTS_PATH}" + ) + + custom_component_dicts = [] + processed_paths = [] + for path in settings_service.settings.COMPONENTS_PATH: + if str(path) in processed_paths: + continue + custom_component_dict = build_langchain_custom_component_list_from_path( + str(path) + ) + custom_component_dicts.append(custom_component_dict) + processed_paths.append(str(path)) + + logger.info(f"Loading {len(custom_component_dicts)} category(ies)") + for custom_component_dict in custom_component_dicts: + # custom_component_dict is a dict of dicts + if not custom_component_dict: + continue + category = list(custom_component_dict.keys())[0] + logger.info( + f"Loading {len(custom_component_dict[category])} component(s) from category {category}" + ) + custom_components_from_file = merge_nested_dicts_with_renaming( + custom_components_from_file, custom_component_dict + ) + + return merge_nested_dicts_with_renaming( + native_components, custom_components_from_file + ) + + +def merge_nested_dicts(dict1, dict2): + for key, value in dict2.items(): + if isinstance(value, dict) and isinstance(dict1.get(key), dict): + dict1[key] = merge_nested_dicts(dict1[key], value) + else: + dict1[key] = value + return dict1