refac: change the way chains are loaded and fix tools

src/backend/langflow/config.yaml

@@ -1,8 +1,8 @@
 chains:
   - LLMChain
   - LLMMathChain
-  - LLMChecker
-  # - ConversationChain
+  - LLMCheckerChain
+  - ConversationChain
 
 agents:
   - ZeroShotAgent
@@ -31,19 +31,16 @@ wrappers:
   - RequestsWrapper
 
 toolkits:
- - OpenAPIToolkit
- - JsonToolkit
+  - OpenAPIToolkit
+  - JsonToolkit
 
 memories:
   - ConversationBufferMemory
 
 embeddings: []
 
-
 vectorstores: []
 
-
 documentloaders: []
 
-
 dev: false

src/backend/langflow/graph/base.py

@@ -82,51 +82,44 @@ class Node:
                 continue
             # If the type is not transformable to a python base class
             # then we need to get the edge that connects to this node
-            if value["type"] == "file":
+            if value.get("type") == "file":
                 # Load the type in value.get('suffixes') using
                 # what is inside value.get('content')
                 # value.get('value') is the file name
-                type_to_load = value.get("suffixes")
                 file_name = value.get("value")
                 content = value.get("content")
+                type_to_load = value.get("suffixes")
                 loaded_dict = load_file(file_name, content, type_to_load)
                 params[key] = loaded_dict
 
             # We should check if the type is in something not
             # the opposite
-            elif value["type"] not in DIRECT_TYPES:
+            elif value.get("type") not in DIRECT_TYPES:
                 # Get the edge that connects to this node
-                try:
-                    edge = next(
-                        (
-                            edge
-                            for edge in self.edges
-                            if edge.target == self
-                            and edge.matched_type in value["type"]
-                        ),
-                        None,
-                    )
+                edges = [
+                    edge
+                    for edge in self.edges
+                    if edge.target == self and edge.matched_type in value["type"]
+                ]
 
-                except Exception as e:
-                    raise e
                 # Get the output of the node that the edge connects to
                 # if the value['list'] is True, then there will be more
                 # than one time setting to params[key]
                 # so we need to append to a list if it exists
                 # or create a new list if it doesn't
 
-                if edge is None and value["required"]:
-                    # break line
+                if value["required"] and not edges:
+                    # If a required parameter is not found, raise an error
                     raise ValueError(
                         f"Required input {key} for module {self.node_type} not found"
                     )
                 elif value["list"]:
-                    if key not in params:
-                        params[key] = []
-                    if edge is not None:
-                        params[key].append(edge.source)
-                elif value["required"] or edge is not None:
-                    params[key] = edge.source
+                    # If this is a list parameter, append all sources to a list
+                    params[key] = [edge.source for edge in edges]
+                elif edges:
+                    # If a single parameter is found, use its source
+                    params[key] = edges[0].source
+
             elif value["required"] or value.get("value"):
                 params[key] = value["value"]
 

src/backend/langflow/graph/graph.py

@@ -17,7 +17,8 @@ from langflow.interface.llms.base import llm_creator
 from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
-from langflow.interface.tools.constants import ALL_TOOLS_NAMES, FILE_TOOLS
+from langflow.interface.tools.constants import FILE_TOOLS
+from langflow.interface.tools.util import get_tools_dict
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.utils import payload
 
@@ -113,7 +114,10 @@ class Graph:
                 nodes.append(AgentNode(node))
             elif node_type in chain_creator.to_list():
                 nodes.append(ChainNode(node))
-            elif node_type in tool_creator.to_list() or node_lc_type in ALL_TOOLS_NAMES:
+            elif (
+                node_type in tool_creator.to_list()
+                or node_lc_type in get_tools_dict().keys()
+            ):
                 if node_type in FILE_TOOLS:
                     nodes.append(FileToolNode(node))
                 nodes.append(ToolNode(node))

src/backend/langflow/interface/agents/custom.py

@@ -1,17 +1,16 @@
 from typing import Any, List, Optional
 
 from langchain import LLMChain
-from langchain.agents import AgentExecutor, ZeroShotAgent
+from langchain.agents import AgentExecutor, Tool, ZeroShotAgent, initialize_agent
 from langchain.agents.agent_toolkits.json.prompt import JSON_PREFIX, JSON_SUFFIX
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
 from langchain.agents.agent_toolkits.pandas.prompt import PREFIX as PANDAS_PREFIX
 from langchain.agents.agent_toolkits.pandas.prompt import SUFFIX as PANDAS_SUFFIX
 from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS
-from langchain.schema import BaseLanguageModel
 from langchain.llms.base import BaseLLM
-from langchain.tools.python.tool import PythonAstREPLTool
-from langchain.agents import initialize_agent, Tool
 from langchain.memory.chat_memory import BaseChatMemory
+from langchain.schema import BaseLanguageModel
+from langchain.tools.python.tool import PythonAstREPLTool
 
 
 class JsonAgent(AgentExecutor):

src/backend/langflow/interface/chains/base.py

@@ -1,10 +1,9 @@
 from typing import Dict, List
 
-from langchain.chains import loading as chains_loading
-
 from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import chain_type_to_cls_dict
 from langflow.settings import settings
-from langflow.utils.util import build_template_from_function
+from langflow.utils.util import build_template_from_class
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 
@@ -15,25 +14,20 @@ class ChainCreator(LangChainTypeCreator):
     @property
     def type_to_loader_dict(self) -> Dict:
         if self.type_dict is None:
-            self.type_dict = chains_loading.type_to_loader_dict
+            self.type_dict = chain_type_to_cls_dict
         return self.type_dict
 
     def get_signature(self, name: str) -> Dict | None:
         try:
-            return build_template_from_function(
-                name, self.type_to_loader_dict, add_function=True
-            )
+            return build_template_from_class(name, chain_type_to_cls_dict)
         except ValueError as exc:
-            raise ValueError("Chain not found") from exc
+            raise ValueError("Memory not found") from exc
 
     def to_list(self) -> List[str]:
         return [
-            chain.__annotations__["return"].__name__
+            chain.__name__
             for chain in self.type_to_loader_dict.values()
-            if (
-                chain.__annotations__["return"].__name__ in settings.chains
-                or settings.dev
-            )
+            if chain.__name__ in settings.chains or settings.dev
         ]
 
 

src/backend/langflow/interface/custom_lists.py

@@ -1,108 +1,21 @@
 from typing import Any
 
 ## LLM
-from langchain import llms, requests
+from langchain import chains, document_loaders, embeddings, llms, memory, requests, vectorstores
 from langchain.agents import agent_toolkits
 from langchain.chat_models import ChatOpenAI
 
-## Memory
-from langchain import memory
-
-## Document Loaders
-from langchain.document_loaders import (
-    AirbyteJSONLoader,
-    AZLyricsLoader,
-    CollegeConfidentialLoader,
-    CoNLLULoader,
-    CSVLoader,
-    DirectoryLoader,
-    EverNoteLoader,
-    FacebookChatLoader,
-    GCSDirectoryLoader,
-    GCSFileLoader,
-    GitbookLoader,
-    GoogleApiClient,
-    GoogleApiYoutubeLoader,
-    GoogleDriveLoader,
-    GutenbergLoader,
-    HNLoader,
-    IFixitLoader,
-    IMSDbLoader,
-    NotebookLoader,
-    NotionDirectoryLoader,
-    ObsidianLoader,
-    OnlinePDFLoader,
-    PagedPDFSplitter,
-    PDFMinerLoader,
-    PyMuPDFLoader,
-    PyPDFLoader,
-    ReadTheDocsLoader,
-    RoamLoader,
-    S3DirectoryLoader,
-    S3FileLoader,
-    SRTLoader,
-    TelegramChatLoader,
-    TextLoader,
-    UnstructuredEmailLoader,
-    UnstructuredFileIOLoader,
-    UnstructuredFileLoader,
-    UnstructuredHTMLLoader,
-    UnstructuredImageLoader,
-    UnstructuredMarkdownLoader,
-    UnstructuredPDFLoader,
-    # BSHTMLLoader,
-    UnstructuredPowerPointLoader,
-    UnstructuredURLLoader,
-    UnstructuredWordDocumentLoader,
-    WebBaseLoader,
-    YoutubeLoader,
-)
-
-## Embeddings
-from langchain.embeddings import (
-    CohereEmbeddings,
-    FakeEmbeddings,
-    HuggingFaceEmbeddings,
-    HuggingFaceHubEmbeddings,
-    HuggingFaceInstructEmbeddings,
-    OpenAIEmbeddings,
-    SelfHostedEmbeddings,
-    SelfHostedHuggingFaceEmbeddings,
-    SelfHostedHuggingFaceInstructEmbeddings,
-    # SagemakerEndpointEmbeddings,
-    TensorflowHubEmbeddings,
-)
-
-## Vector Stores
-from langchain.vectorstores import (
-    FAISS,
-    AtlasDB,
-    Chroma,
-    DeepLake,
-    ElasticVectorSearch,
-    Milvus,
-    OpenSearchVectorSearch,
-    Pinecone,
-    Qdrant,
-    VectorStore,
-    Weaviate,
-)
-
-## Toolkits
 from langflow.interface.importing.utils import import_class
 
 ## LLM
-
 llm_type_to_cls_dict = llms.type_to_cls_dict
 llm_type_to_cls_dict["openai-chat"] = ChatOpenAI  # type: ignore
 
-
 ## Chain
-# from langchain.chains.loading import type_to_loader_dict
-# from langchain.chains.conversation.base import ConversationChain
-
-# chain_type_to_cls_dict = type_to_loader_dict
-# chain_type_to_cls_dict["conversation_chain"] = ConversationChain
+chain_type_to_cls_dict: dict[str, Any] = {
+    chain_name: import_class(f"langchain.chains.{chain_name}")
+    for chain_name in chains.__all__
+}
 
 toolkit_type_to_loader_dict: dict[str, Any] = {
     toolkit_name: import_class(f"langchain.agents.agent_toolkits.{toolkit_name}")
@@ -118,99 +31,33 @@ toolkit_type_to_cls_dict: dict[str, Any] = {
     if not toolkit_name.islower()
 }
 
-## Memory
-
-
+## Memories
 memory_type_to_cls_dict: dict[str, Any] = {
     memory_name: import_class(f"langchain.memory.{memory_name}")
     for memory_name in memory.__all__
 }
 
-
+## Wrappers
 wrapper_type_to_cls_dict: dict[str, Any] = {
     wrapper.__name__: wrapper for wrapper in [requests.RequestsWrapper]
 }
 
 ## Embeddings
-
-embedding_type_to_cls_dict = {
-    "OpenAIEmbeddings": OpenAIEmbeddings,
-    "HuggingFaceEmbeddings": HuggingFaceEmbeddings,
-    "CohereEmbeddings": CohereEmbeddings,
-    "HuggingFaceHubEmbeddings": HuggingFaceHubEmbeddings,
-    "TensorflowHubEmbeddings": TensorflowHubEmbeddings,
-    # "SagemakerEndpointEmbeddings": SagemakerEndpointEmbeddings,
-    "HuggingFaceInstructEmbeddings": HuggingFaceInstructEmbeddings,
-    "SelfHostedEmbeddings": SelfHostedEmbeddings,
-    "SelfHostedHuggingFaceEmbeddings": SelfHostedHuggingFaceEmbeddings,
-    "SelfHostedHuggingFaceInstructEmbeddings": SelfHostedHuggingFaceInstructEmbeddings,
-    "FakeEmbeddings": FakeEmbeddings,
+embedding_type_to_cls_dict: dict[str, Any] = {
+    embedding_name: import_class(f"langchain.embeddings.{embedding_name}")
+    for embedding_name in embeddings.__all__
 }
 
 ## Vector Stores
-
-vectorstores_type_to_cls_dict = {
-    "ElasticVectorSearch": ElasticVectorSearch,
-    "FAISS": FAISS,
-    "VectorStore": VectorStore,
-    "Pinecone": Pinecone,
-    "Weaviate": Weaviate,
-    "Qdrant": Qdrant,
-    "Milvus": Milvus,
-    "Chroma": Chroma,
-    "OpenSearchVectorSearch": OpenSearchVectorSearch,
-    "AtlasDB": AtlasDB,
-    "DeepLake": DeepLake,
+vectorstores_type_to_cls_dict: dict[str, Any] = {
+    vectorstore_name: import_class(f"langchain.vectorstores.{vectorstore_name}")
+    for vectorstore_name in vectorstores.__all__
 }
 
 ## Document Loaders
-
-documentloaders_type_to_cls_dict = {
-    "UnstructuredFileLoader": UnstructuredFileLoader,
-    "UnstructuredFileIOLoader": UnstructuredFileIOLoader,
-    "UnstructuredURLLoader": UnstructuredURLLoader,
-    "DirectoryLoader": DirectoryLoader,
-    "NotionDirectoryLoader": NotionDirectoryLoader,
-    "ReadTheDocsLoader": ReadTheDocsLoader,
-    "GoogleDriveLoader": GoogleDriveLoader,
-    "UnstructuredHTMLLoader": UnstructuredHTMLLoader,
-    # "BSHTMLLoader": BSHTMLLoader,
-    "UnstructuredPowerPointLoader": UnstructuredPowerPointLoader,
-    "UnstructuredWordDocumentLoader": UnstructuredWordDocumentLoader,
-    "UnstructuredPDFLoader": UnstructuredPDFLoader,
-    "UnstructuredImageLoader": UnstructuredImageLoader,
-    "ObsidianLoader": ObsidianLoader,
-    "UnstructuredEmailLoader": UnstructuredEmailLoader,
-    "UnstructuredMarkdownLoader": UnstructuredMarkdownLoader,
-    "RoamLoader": RoamLoader,
-    "YoutubeLoader": YoutubeLoader,
-    "S3FileLoader": S3FileLoader,
-    "TextLoader": TextLoader,
-    "HNLoader": HNLoader,
-    "GitbookLoader": GitbookLoader,
-    "S3DirectoryLoader": S3DirectoryLoader,
-    "GCSFileLoader": GCSFileLoader,
-    "GCSDirectoryLoader": GCSDirectoryLoader,
-    "WebBaseLoader": WebBaseLoader,
-    "IMSDbLoader": IMSDbLoader,
-    "AZLyricsLoader": AZLyricsLoader,
-    "CollegeConfidentialLoader": CollegeConfidentialLoader,
-    "IFixitLoader": IFixitLoader,
-    "GutenbergLoader": GutenbergLoader,
-    "PagedPDFSplitter": PagedPDFSplitter,
-    "PyPDFLoader": PyPDFLoader,
-    "EverNoteLoader": EverNoteLoader,
-    "AirbyteJSONLoader": AirbyteJSONLoader,
-    "OnlinePDFLoader": OnlinePDFLoader,
-    "PDFMinerLoader": PDFMinerLoader,
-    "PyMuPDFLoader": PyMuPDFLoader,
-    "TelegramChatLoader": TelegramChatLoader,
-    "SRTLoader": SRTLoader,
-    "FacebookChatLoader": FacebookChatLoader,
-    "NotebookLoader": NotebookLoader,
-    "CoNLLULoader": CoNLLULoader,
-    "GoogleApiYoutubeLoader": GoogleApiYoutubeLoader,
-    "GoogleApiClient": GoogleApiClient,
-    "CSVLoader": CSVLoader,
-    # "BlackboardLoader",
+documentloaders_type_to_cls_dict: dict[str, Any] = {
+    documentloader_name: import_class(
+        f"langchain.document_loaders.{documentloader_name}"
+    )
+    for documentloader_name in document_loaders.__all__
 }

src/backend/langflow/interface/importing/utils.py

@@ -6,9 +6,10 @@ from typing import Any
 from langchain import PromptTemplate
 from langchain.agents import Agent
 from langchain.chains.base import Chain
+from langchain.chat_models.base import BaseChatModel
 from langchain.llms.base import BaseLLM
 from langchain.tools import BaseTool
-from langchain.chat_models.base import BaseChatModel
+
 from langflow.interface.tools.util import get_tool_by_name
 
 

src/backend/langflow/interface/prompts/custom.py

@@ -1,11 +1,11 @@
 from typing import List, Optional
 
 from langchain.prompts import PromptTemplate
+from pydantic import root_validator
+
 from langflow.graph.utils import extract_input_variables_from_prompt
 from langflow.template.base import Template, TemplateField
 from langflow.template.nodes import PromptTemplateNode
-from pydantic import root_validator
-
 
 CHARACTER_PROMPT = """I want you to act like {character} from {series}.
 I want you to respond and answer like {character}. do not write any explanations. only answer like {character}.

src/backend/langflow/interface/run.py

@@ -1,5 +1,6 @@
 import contextlib
 import io
+import logging
 import re
 from typing import Any, Dict
 
@@ -7,7 +8,6 @@ from langflow.cache.utils import compute_hash, load_cache, save_cache
 from langflow.graph.graph import Graph
 from langflow.interface import loading
 from langflow.utils import payload
-import logging
 
 logger = logging.getLogger(__name__)
 

src/backend/langflow/interface/tools/base.py

@@ -10,7 +10,6 @@ from langchain.agents.load_tools import (
 from langflow.custom import customs
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.tools.constants import (
-    ALL_TOOLS_NAMES,
     CUSTOM_TOOLS,
     FILE_TOOLS,
 )
@@ -133,8 +132,8 @@ class ToolCreator(LangChainTypeCreator):
 
         tools = []
 
-        for tool in ALL_TOOLS_NAMES:
-            tool_params = get_tool_params(get_tool_by_name(tool))
+        for tool, fcn in get_tools_dict().items():
+            tool_params = get_tool_params(fcn)
 
             if tool_params and not tool_params.get("name"):
                 tool_params["name"] = tool
@@ -145,9 +144,7 @@ class ToolCreator(LangChainTypeCreator):
             ):
                 tools.append(tool_params["name"])
 
-        # Add Tool
-        custom_tools = customs.get_custom_nodes("tools")
-        return tools + list(custom_tools.keys())
+        return tools
 
 
 tool_creator = ToolCreator()

src/backend/langflow/interface/tools/constants.py

@@ -1,11 +1,21 @@
 from langchain.agents import Tool
-from langchain.agents.load_tools import get_all_tool_names
+from langchain.agents.load_tools import (
+    _BASE_TOOLS,
+    _EXTRA_LLM_TOOLS,
+    _EXTRA_OPTIONAL_TOOLS,
+    _LLM_TOOLS,
+)
 from langchain.tools.json.tool import JsonSpec
 
 from langflow.interface.custom.types import PythonFunction
 
 FILE_TOOLS = {"JsonSpec": JsonSpec}
 CUSTOM_TOOLS = {"Tool": Tool, "PythonFunction": PythonFunction}
-ALL_TOOLS_NAMES = set(
-    get_all_tool_names() + list(CUSTOM_TOOLS.keys()) + list(FILE_TOOLS.keys())
-)
+ALL_TOOLS_NAMES = {
+    **_BASE_TOOLS,
+    **_LLM_TOOLS,  # type: ignore
+    **{k: v[0] for k, v in _EXTRA_LLM_TOOLS.items()},  # type: ignore
+    **{k: v[0] for k, v in _EXTRA_OPTIONAL_TOOLS.items()},
+    **CUSTOM_TOOLS,
+    **FILE_TOOLS,  # type: ignore
+}

src/backend/langflow/interface/tools/util.py

@@ -2,28 +2,22 @@ import ast
 import inspect
 from typing import Dict, Union
 
-from langchain.agents.load_tools import (
-    _BASE_TOOLS,
-    _EXTRA_LLM_TOOLS,
-    _EXTRA_OPTIONAL_TOOLS,
-    _LLM_TOOLS,
-)
 from langchain.agents.tools import Tool
 
-from langflow.interface.tools.constants import CUSTOM_TOOLS, FILE_TOOLS
+from langflow.interface.tools.constants import ALL_TOOLS_NAMES
 
 
 def get_tools_dict():
     """Get the tools dictionary."""
 
-    return {
-        **_BASE_TOOLS,
-        **_LLM_TOOLS,
-        **{k: v[0] for k, v in _EXTRA_LLM_TOOLS.items()},
-        **{k: v[0] for k, v in _EXTRA_OPTIONAL_TOOLS.items()},
-        **CUSTOM_TOOLS,
-        **FILE_TOOLS,
-    }
+    all_tools = {}
+
+    for tool, fcn in ALL_TOOLS_NAMES.items():
+        if tool_params := get_tool_params(fcn):
+            tool_name = tool_params.get("name") or str(tool)
+            all_tools[tool_name] = fcn
+
+    return all_tools
 
 
 def get_tool_by_name(name: str):

src/backend/langflow/template/nodes.py

@@ -1,9 +1,10 @@
 from typing import Optional
+
+from langchain.agents import loading
 from langchain.agents.mrkl import prompt
 
 from langflow.template.base import FrontendNode, Template, TemplateField
 from langflow.utils.constants import DEFAULT_PYTHON_FUNCTION
-from langchain.agents import loading
 
 
 class ZeroShotPromptNode(FrontendNode):