VectorStores, DocumentLoaders, TextSplitters, Embeddings and other additions (#157)

This release adds many bug fixes and exposes many other new nodes.

.gitignore

@@ -9,6 +9,11 @@ lerna-debug.log*
 # Mac
 .DS_Store
 
+# VSCode
+.vscode
+.chroma
+.ruff_cache
+
 # Diagnostic reports (https://nodejs.org/api/report.html)
 report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 

README.md

@@ -49,7 +49,7 @@ flow("Hey, have you heard of LangFlow?")
 
 ## 👋 Contributing
 
-We welcome contributions from developers of all levels to our open-source project on GitHub. If you'd like to contribute, please check our contributing guidelines and help make LangFlow more accessible.
+We welcome contributions from developers of all levels to our open-source project on GitHub. If you'd like to contribute, please check our [contributing guidelines](./CONTRIBUTING.md) and help make LangFlow more accessible.
 
 
 [![Star History Chart](https://api.star-history.com/svg?repos=logspace-ai/langflow&type=Timeline)](https://star-history.com/#logspace-ai/langflow&Date)

dev.Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.10-slim
 WORKDIR /app
 
 # Install Poetry
-RUN apt-get update && apt-get install gcc curl -y
+RUN apt-get update && apt-get install gcc g++ curl build-essential postgresql-server-dev-all -y
 RUN curl -sSL https://install.python-poetry.org | python3 -
 # # Add Poetry to PATH
 ENV PATH="${PATH}:/root/.local/bin"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "langflow"
-version = "0.0.55"
+version = "0.0.56"
 description = "A Python package with a built-in web application"
 authors = ["Logspace <contact@logspace.ai>"]
 maintainers = [
@@ -34,8 +34,18 @@ openai = "^0.27.2"
 types-pyyaml = "^6.0.12.8"
 dill = "^0.3.6"
 pandas = "^1.5.3"
+chromadb = "^0.3.21"
 huggingface-hub = "^0.13.3"
 rich = "^13.3.3"
+llama-cpp-python = "0.1.23"
+networkx = "^3.1"
+unstructured = "^0.5.11"
+pypdf = "^3.7.1"
+lxml = "^4.9.2"
+pysrt = "^1.1.2"
+fake-useragent = "^1.1.3"
+docstring-parser = "^0.15"
+psycopg2 = "^2.9.6"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"

src/backend/langflow/api/endpoints.py

@@ -3,7 +3,7 @@ from typing import Any, Dict
 
 from fastapi import APIRouter, HTTPException
 
-from langflow.interface.run import process_graph
+from langflow.interface.run import process_graph_cached
 from langflow.interface.types import build_langchain_types_dict
 
 # build router
@@ -19,7 +19,7 @@ def get_all():
 @router.post("/predict")
 def get_load(data: Dict[str, Any]):
     try:
-        return process_graph(data)
+        return process_graph_cached(data)
     except Exception as e:
         # Log stack trace
         logger.exception(e)

src/backend/langflow/cache/utils.py

@@ -1,18 +1,62 @@
+import base64
 import contextlib
+import functools
 import hashlib
 import json
 import os
 import tempfile
+from collections import OrderedDict
 from pathlib import Path
 
 import dill  # type: ignore
 
+
+def create_cache_folder(func):
+    def wrapper(*args, **kwargs):
+        # Get the destination folder
+        cache_path = Path(tempfile.gettempdir()) / PREFIX
+
+        # Create the destination folder if it doesn't exist
+        os.makedirs(cache_path, exist_ok=True)
+
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def memoize_dict(maxsize=128):
+    cache = OrderedDict()
+
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            hashed = compute_dict_hash(args[0])
+            key = (func.__name__, hashed, frozenset(kwargs.items()))
+            if key not in cache:
+                result = func(*args, **kwargs)
+                cache[key] = result
+                if len(cache) > maxsize:
+                    cache.popitem(last=False)
+            else:
+                result = cache[key]
+            return result
+
+        def clear_cache():
+            cache.clear()
+
+        wrapper.clear_cache = clear_cache
+        return wrapper
+
+    return decorator
+
+
 PREFIX = "langflow_cache"
 
 
+@create_cache_folder
 def clear_old_cache_files(max_cache_size: int = 3):
-    cache_dir = Path(tempfile.gettempdir())
+    cache_dir = Path(tempfile.gettempdir()) / PREFIX
-    cache_files = list(cache_dir.glob(f"{PREFIX}_*.dill"))
+    cache_files = list(cache_dir.glob("*.dill"))
 
     if len(cache_files) > max_cache_size:
         cache_files_sorted_by_mtime = sorted(
@@ -24,6 +68,13 @@ def clear_old_cache_files(max_cache_size: int = 3):
                 os.remove(cache_file)
 
 
+def compute_dict_hash(graph_data):
+    graph_data = filter_json(graph_data)
+
+    cleaned_graph_json = json.dumps(graph_data, sort_keys=True)
+    return hashlib.sha256(cleaned_graph_json.encode("utf-8")).hexdigest()
+
+
 def filter_json(json_data):
     filtered_data = json_data.copy()
 
@@ -48,15 +99,40 @@ def filter_json(json_data):
     return filtered_data
 
 
-def compute_hash(graph_data):
+@create_cache_folder
-    graph_data = filter_json(graph_data)
+def save_binary_file(content: str, file_name: str, accepted_types: list[str]) -> str:
+    """
+    Save a binary file to the specified folder.
 
-    cleaned_graph_json = json.dumps(graph_data, sort_keys=True)
+    Args:
-    return hashlib.sha256(cleaned_graph_json.encode("utf-8")).hexdigest()
+        content: The content of the file as a bytes object.
+        file_name: The name of the file, including its extension.
+
+    Returns:
+        The path to the saved file.
+    """
+    if not any(file_name.endswith(suffix) for suffix in accepted_types):
+        raise ValueError(f"File {file_name} is not accepted")
+
+    # Get the destination folder
+    cache_path = Path(tempfile.gettempdir()) / PREFIX
+
+    data = content.split(",")[1]
+    decoded_bytes = base64.b64decode(data)
+
+    # Create the full file path
+    file_path = os.path.join(cache_path, file_name)
+
+    # Save the binary content to the file
+    with open(file_path, "wb") as file:
+        file.write(decoded_bytes)
+
+    return file_path
 
 
+@create_cache_folder
 def save_cache(hash_val: str, chat_data, clean_old_cache_files: bool):
-    cache_path = Path(tempfile.gettempdir()) / f"{PREFIX}_{hash_val}.dill"
+    cache_path = Path(tempfile.gettempdir()) / PREFIX / f"{hash_val}.dill"
     with cache_path.open("wb") as cache_file:
         dill.dump(chat_data, cache_file)
 
@@ -64,8 +140,9 @@ def save_cache(hash_val: str, chat_data, clean_old_cache_files: bool):
         clear_old_cache_files()
 
 
+@create_cache_folder
 def load_cache(hash_val):
-    cache_path = Path(tempfile.gettempdir()) / f"{PREFIX}_{hash_val}.dill"
+    cache_path = Path(tempfile.gettempdir()) / PREFIX / f"{hash_val}.dill"
     if cache_path.exists():
         with cache_path.open("rb") as cache_file:
             return dill.load(cache_file)

src/backend/langflow/config.yaml

@@ -12,6 +12,9 @@ agents:
   - JsonAgent
   - CSVAgent
   - initialize_agent
+  - VectorStoreAgent
+  - VectorStoreRouterAgent
+  - SQLAgent
 
 prompts:
   - PromptTemplate
@@ -27,6 +30,7 @@ llms:
   # - AzureOpenAI
   - ChatOpenAI
   - HuggingFaceHub
+  - LlamaCpp
 
 tools:
   - Search
@@ -43,16 +47,48 @@ wrappers:
 toolkits:
   - OpenAPIToolkit
   - JsonToolkit
+  - VectorStoreInfo
+  - VectorStoreRouterToolkit
 
 memories:
   - ConversationBufferMemory
   - ConversationSummaryMemory
   - ConversationKGMemory
 
-embeddings: []
+embeddings:
+  - OpenAIEmbeddings
 
-vectorstores: []
+vectorstores:
+  - Chroma
 
-documentloaders: []
+documentloaders:
+  - AirbyteJSONLoader
+  - CoNLLULoader
+  - CSVLoader
+  - UnstructuredEmailLoader
+  - EverNoteLoader
+  - FacebookChatLoader
+  - GutenbergLoader
+  - BSHTMLLoader
+  - UnstructuredHTMLLoader
+  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
+  - UnstructuredMarkdownLoader
+  - PyPDFLoader
+  - UnstructuredPowerPointLoader
+  - SRTLoader
+  - TelegramChatLoader
+  - TextLoader
+  - UnstructuredWordDocumentLoader
+  - WebBaseLoader
+  - AZLyricsLoader
+  - CollegeConfidentialLoader
+  - HNLoader
+  - IFixitLoader
+  - IMSDbLoader
+  - GitbookLoader
+  - ReadTheDocsLoader
+
+textsplitters:
+  - CharacterTextSplitter
 
 dev: false

src/backend/langflow/custom/customs.py

@@ -8,6 +8,9 @@ CUSTOM_NODES = {
         "JsonAgent": nodes.JsonAgentNode(),
         "CSVAgent": nodes.CSVAgentNode(),
         "initialize_agent": nodes.InitializeAgentNode(),
+        "VectorStoreAgent": nodes.VectorStoreAgentNode(),
+        "VectorStoreRouterAgent": nodes.VectorStoreRouterAgentNode(),
+        "SQLAgent": nodes.SQLAgentNode(),
     },
 }
 

src/backend/langflow/graph/base.py

@@ -3,13 +3,14 @@
 #   - Defer prompts building to the last moment or when they have all the tools
 #   - Build each inner agent first, then build the outer agent
 
+import contextlib
 import types
 import warnings
 from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
+from langflow.cache import utils as cache_utils
 from langflow.graph.constants import DIRECT_TYPES
-from langflow.graph.utils import load_file
 from langflow.interface import loading
 from langflow.interface.listing import ALL_TYPES_DICT
 from langflow.utils.logger import logger
@@ -88,11 +89,12 @@ class Node:
                 file_name = value.get("value")
                 content = value.get("content")
                 type_to_load = value.get("suffixes")
-                loaded_dict = load_file(file_name, content, type_to_load)
+                file_path = cache_utils.save_binary_file(
-                params[key] = loaded_dict
+                    content=content, file_name=file_name, accepted_types=type_to_load
+                )
+
+                params[key] = file_path
 
-            # We should check if the type is in something not
-            # the opposite
             elif value.get("type") not in DIRECT_TYPES:
                 # Get the edge that connects to this node
                 edges = [
@@ -126,6 +128,9 @@ class Node:
                 new_value = value.get("value")
                 if new_value is None:
                     warnings.warn(f"Value for {key} in {self.node_type} is None. ")
+                if value.get("type") == "int":
+                    with contextlib.suppress(TypeError, ValueError):
+                        new_value = int(new_value)  # type: ignore
                 params[key] = new_value
 
         # Add _type to params
@@ -160,6 +165,7 @@ class Node:
                         result = result.run  # type: ignore
                     elif hasattr(result, "get_function"):
                         result = result.get_function()  # type: ignore
+
                 self.params[key] = result
             elif isinstance(value, list) and all(
                 isinstance(node, Node) for node in value
@@ -189,6 +195,15 @@ class Node:
     def build(self, force: bool = False) -> Any:
         if not self._built or force:
             self._build()
+
+        #! Deepcopy is breaking for vectorstores
+        if self.base_type in [
+            "vectorstores",
+            "VectorStoreRouterAgent",
+            "VectorStoreAgent",
+            "VectorStoreInfo",
+        ] or self.node_type in ["VectorStoreInfo", "VectorStoreRouterToolkit"]:
+            return self._built_object
         return deepcopy(self._built_object)
 
     def add_edge(self, edge: "Edge") -> None:

src/backend/langflow/graph/graph.py

@@ -4,22 +4,30 @@ from langflow.graph.base import Edge, Node
 from langflow.graph.nodes import (
     AgentNode,
     ChainNode,
+    DocumentLoaderNode,
+    EmbeddingNode,
     FileToolNode,
     LLMNode,
     MemoryNode,
     PromptNode,
+    TextSplitterNode,
     ToolkitNode,
     ToolNode,
+    VectorStoreNode,
     WrapperNode,
 )
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
+from langflow.interface.document_loaders.base import documentloader_creator
+from langflow.interface.embeddings.base import embedding_creator
 from langflow.interface.llms.base import llm_creator
 from langflow.interface.memories.base import memory_creator
 from langflow.interface.prompts.base import prompt_creator
+from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
 from langflow.interface.tools.constants import FILE_TOOLS
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.utils import payload
 
@@ -117,6 +125,10 @@ class Graph:
             **{t: WrapperNode for t in wrapper_creator.to_list()},
             **{t: LLMNode for t in llm_creator.to_list()},
             **{t: MemoryNode for t in memory_creator.to_list()},
+            **{t: EmbeddingNode for t in embedding_creator.to_list()},
+            **{t: VectorStoreNode for t in vectorstore_creator.to_list()},
+            **{t: DocumentLoaderNode for t in documentloader_creator.to_list()},
+            **{t: TextSplitterNode for t in textsplitter_creator.to_list()},
         }
 
         if node_type in FILE_TOOLS:

src/backend/langflow/graph/nodes.py

@@ -32,6 +32,10 @@ class AgentNode(Node):
                 chain_node.build(tools=self.tools)
 
             self._build()
+
+        #! Cannot deepcopy VectorStore, VectorStoreRouter, or SQL agents
+        if self.node_type in ["VectorStoreAgent", "VectorStoreRouterAgent", "SQLAgent"]:
+            return self._built_object
         return deepcopy(self._built_object)
 
 
@@ -39,11 +43,6 @@ class ToolNode(Node):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="tools")
 
-    def build(self, force: bool = False) -> Any:
-        if not self._built or force:
-            self._build()
-        return deepcopy(self._built_object)
-
 
 class PromptNode(Node):
     def __init__(self, data: Dict):
@@ -109,32 +108,16 @@ class LLMNode(Node):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="llms")
 
-    def build(self, force: bool = False) -> Any:
-        if not self._built or force:
-            self._build()
-        return deepcopy(self._built_object)
-
 
 class ToolkitNode(Node):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="toolkits")
 
-    def build(self, force: bool = False) -> Any:
-        if not self._built or force:
-            self._build()
-
-        return deepcopy(self._built_object)
-
 
 class FileToolNode(ToolNode):
     def __init__(self, data: Dict):
         super().__init__(data)
 
-    def build(self, force: bool = False) -> Any:
-        if not self._built or force:
-            self._build()
-        return deepcopy(self._built_object)
-
 
 class WrapperNode(Node):
     def __init__(self, data: Dict):
@@ -148,11 +131,26 @@ class WrapperNode(Node):
         return deepcopy(self._built_object)
 
 
+class DocumentLoaderNode(Node):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="documentloaders")
+
+
+class EmbeddingNode(Node):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="embeddings")
+
+
+class VectorStoreNode(Node):
+    def __init__(self, data: Dict):
+        super().__init__(data, base_type="vectorstores")
+
+
 class MemoryNode(Node):
     def __init__(self, data: Dict):
         super().__init__(data, base_type="memory")
 
-    def build(self, force: bool = False) -> Any:
+
-        if not self._built or force:
+class TextSplitterNode(Node):
-            self._build()
+    def __init__(self, data: Dict):
-        return deepcopy(self._built_object)
+        super().__init__(data, base_type="textsplitters")

src/backend/langflow/graph/utils.py

@@ -1,45 +1,4 @@
-import base64
-import csv
-import io
-import json
 import re
-from typing import Any
-
-import yaml
-
-
-def load_file(file_name, file_content, accepted_types) -> Any:
-    """Load a file from a string."""
-    # Check if the file is accepted
-    if not any(file_name.endswith(suffix) for suffix in accepted_types):
-        raise ValueError(f"File {file_name} is not accepted")
-    # Get the suffix
-    suffix = file_name.split(".")[-1]
-    # file_content == 'data:application/x-yaml;base64,b3BlbmFwaTogIjMuMC4wIg...'
-    data = file_content.split(",")[1]
-    decoded_bytes = base64.b64decode(data)
-
-    # Convert the bytes object to a string
-    decoded_string = decoded_bytes.decode("utf-8")
-    if suffix == "json":
-        # Return the json content
-        return json.loads(decoded_string)
-    elif suffix in ["yaml", "yml"]:
-        # Return the yaml content
-        loaded_yaml = yaml.load(decoded_string, Loader=yaml.FullLoader)
-        try:
-            from langchain.agents.agent_toolkits.openapi.spec import reduce_openapi_spec  # type: ignore
-
-            return reduce_openapi_spec(loaded_yaml)
-        except ImportError:
-            return loaded_yaml
-
-    elif suffix == "csv":
-        # Load the csv content
-        csv_reader = csv.DictReader(io.StringIO(decoded_string))
-        return list(csv_reader)
-    else:
-        raise ValueError(f"File {file_name} is not accepted")
 
 
 def validate_prompt(prompt: str):

src/backend/langflow/interface/agents/base.py

@@ -6,6 +6,7 @@ from langflow.custom.customs import get_custom_nodes
 from langflow.interface.agents.custom import CUSTOM_AGENTS
 from langflow.interface.base import LangChainTypeCreator
 from langflow.settings import settings
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -31,6 +32,9 @@ class AgentCreator(LangChainTypeCreator):
             )
         except ValueError as exc:
             raise ValueError("Agent not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Agent {name} not loaded: {exc}")
+            return None
 
     # Now this is a generator
     def to_list(self) -> List[str]:

src/backend/langflow/interface/agents/custom.py

@@ -1,16 +1,37 @@
 from typing import Any, List, Optional
 
 from langchain import LLMChain
-from langchain.agents import AgentExecutor, Tool, ZeroShotAgent, initialize_agent
+from langchain.agents import (
+    AgentExecutor,
+    Tool,
+    ZeroShotAgent,
+    initialize_agent,
+)
+from langchain.agents.agent_toolkits import (
+    SQLDatabaseToolkit,
+    VectorStoreInfo,
+    VectorStoreRouterToolkit,
+    VectorStoreToolkit,
+)
 from langchain.agents.agent_toolkits.json.prompt import JSON_PREFIX, JSON_SUFFIX
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
 from langchain.agents.agent_toolkits.pandas.prompt import PREFIX as PANDAS_PREFIX
 from langchain.agents.agent_toolkits.pandas.prompt import SUFFIX as PANDAS_SUFFIX
+from langchain.agents.agent_toolkits.sql.prompt import SQL_PREFIX, SQL_SUFFIX
+from langchain.agents.agent_toolkits.vectorstore.prompt import (
+    PREFIX as VECTORSTORE_PREFIX,
+)
+from langchain.agents.agent_toolkits.vectorstore.prompt import (
+    ROUTER_PREFIX as VECTORSTORE_ROUTER_PREFIX,
+)
 from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS
+from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS as SQL_FORMAT_INSTRUCTIONS
 from langchain.llms.base import BaseLLM
 from langchain.memory.chat_memory import BaseChatMemory
 from langchain.schema import BaseLanguageModel
+from langchain.sql_database import SQLDatabase
 from langchain.tools.python.tool import PythonAstREPLTool
+from langchain.tools.sql_database.prompt import QUERY_CHECKER
 
 
 class JsonAgent(AgentExecutor):
@@ -66,7 +87,7 @@ class CSVAgent(AgentExecutor):
     @classmethod
     def from_toolkit_and_llm(
         cls,
-        path: dict,
+        path: str,
         llm: BaseLanguageModel,
         pandas_kwargs: Optional[dict] = None,
         **kwargs: Any
@@ -74,7 +95,7 @@ class CSVAgent(AgentExecutor):
         import pandas as pd  # type: ignore
 
         _kwargs = pandas_kwargs or {}
-        df = pd.DataFrame.from_dict(path, **_kwargs)
+        df = pd.read_csv(path, **_kwargs)
 
         tools = [PythonAstREPLTool(locals={"df": df})]  # type: ignore
         prompt = ZeroShotAgent.create_prompt(
@@ -97,6 +118,153 @@ class CSVAgent(AgentExecutor):
         return super().run(*args, **kwargs)
 
 
+class VectorStoreAgent(AgentExecutor):
+    """Vector Store agent"""
+
+    @staticmethod
+    def function_name():
+        return "VectorStoreAgent"
+
+    @classmethod
+    def initialize(cls, *args, **kwargs):
+        return cls.from_toolkit_and_llm(*args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_toolkit_and_llm(
+        cls, llm: BaseLLM, vectorstoreinfo: VectorStoreInfo, **kwargs: Any
+    ):
+        """Construct a vectorstore agent from an LLM and tools."""
+
+        toolkit = VectorStoreToolkit(vectorstore_info=vectorstoreinfo, llm=llm)
+
+        tools = toolkit.get_tools()
+        prompt = ZeroShotAgent.create_prompt(tools, prefix=VECTORSTORE_PREFIX)
+        llm_chain = LLMChain(
+            llm=llm,
+            prompt=prompt,
+        )
+        tool_names = [tool.name for tool in tools]
+        agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names, **kwargs)
+        return AgentExecutor.from_agent_and_tools(
+            agent=agent, tools=tools, verbose=True
+        )
+
+    def run(self, *args, **kwargs):
+        return super().run(*args, **kwargs)
+
+
+class SQLAgent(AgentExecutor):
+    """SQL agent"""
+
+    @staticmethod
+    def function_name():
+        return "SQLAgent"
+
+    @classmethod
+    def initialize(cls, *args, **kwargs):
+        return cls.from_toolkit_and_llm(*args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_toolkit_and_llm(cls, llm: BaseLLM, database_uri: str, **kwargs: Any):
+        """Construct a sql agent from an LLM and tools."""
+        db = SQLDatabase.from_uri(database_uri)
+        toolkit = SQLDatabaseToolkit(db=db)
+
+        # The right code should be this, but there is a problem with tools = toolkit.get_tools()
+        # related to `OPENAI_API_KEY`
+        # return create_sql_agent(llm=llm, toolkit=toolkit, verbose=True)
+        from langchain.prompts import PromptTemplate
+        from langchain.tools.sql_database.tool import (
+            InfoSQLDatabaseTool,
+            ListSQLDatabaseTool,
+            QueryCheckerTool,
+            QuerySQLDataBaseTool,
+        )
+
+        llmchain = LLMChain(
+            llm=llm,
+            prompt=PromptTemplate(
+                template=QUERY_CHECKER, input_variables=["query", "dialect"]
+            ),
+        )
+
+        tools = [
+            QuerySQLDataBaseTool(db=db),  # type: ignore
+            InfoSQLDatabaseTool(db=db),  # type: ignore
+            ListSQLDatabaseTool(db=db),  # type: ignore
+            QueryCheckerTool(db=db, llm_chain=llmchain),  # type: ignore
+        ]
+
+        prefix = SQL_PREFIX.format(dialect=toolkit.dialect, top_k=10)
+        prompt = ZeroShotAgent.create_prompt(
+            tools=tools,  # type: ignore
+            prefix=prefix,
+            suffix=SQL_SUFFIX,
+            format_instructions=SQL_FORMAT_INSTRUCTIONS,
+        )
+        llm_chain = LLMChain(
+            llm=llm,
+            prompt=prompt,
+        )
+        tool_names = [tool.name for tool in tools]  # type: ignore
+        agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names, **kwargs)
+        return AgentExecutor.from_agent_and_tools(
+            agent=agent,
+            tools=tools,  # type: ignore
+            verbose=True,
+            max_iterations=15,
+            early_stopping_method="force",
+        )
+
+    def run(self, *args, **kwargs):
+        return super().run(*args, **kwargs)
+
+
+class VectorStoreRouterAgent(AgentExecutor):
+    """Vector Store Router Agent"""
+
+    @staticmethod
+    def function_name():
+        return "VectorStoreRouterAgent"
+
+    @classmethod
+    def initialize(cls, *args, **kwargs):
+        return cls.from_toolkit_and_llm(*args, **kwargs)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def from_toolkit_and_llm(
+        cls,
+        llm: BaseLanguageModel,
+        vectorstoreroutertoolkit: VectorStoreRouterToolkit,
+        **kwargs: Any
+    ):
+        """Construct a vector store router agent from an LLM and tools."""
+
+        tools = vectorstoreroutertoolkit.get_tools()
+        prompt = ZeroShotAgent.create_prompt(tools, prefix=VECTORSTORE_ROUTER_PREFIX)
+        llm_chain = LLMChain(
+            llm=llm,
+            prompt=prompt,
+        )
+        tool_names = [tool.name for tool in tools]
+        agent = ZeroShotAgent(llm_chain=llm_chain, allowed_tools=tool_names, **kwargs)
+        return AgentExecutor.from_agent_and_tools(
+            agent=agent, tools=tools, verbose=True
+        )
+
+    def run(self, *args, **kwargs):
+        return super().run(*args, **kwargs)
+
+
 class InitializeAgent(AgentExecutor):
     """Implementation of initialize_agent function"""
 
@@ -128,4 +296,7 @@ CUSTOM_AGENTS = {
     "JsonAgent": JsonAgent,
     "CSVAgent": CSVAgent,
     "initialize_agent": InitializeAgent,
+    "VectorStoreAgent": VectorStoreAgent,
+    "VectorStoreRouterAgent": VectorStoreRouterAgent,
+    "SQLAgent": SQLAgent,
 }

src/backend/langflow/interface/base.py

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Type, Union
 from pydantic import BaseModel
 
 from langflow.template.base import FrontendNode, Template, TemplateField
+from langflow.utils.logger import logger
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 
@@ -39,14 +40,18 @@ class LangChainTypeCreator(BaseModel, ABC):
             # frontend_node.to_dict() returns a dict with the following structure:
             # {name: {template: {fields}, description: str}}
             # so we should update the result dict
-            result[self.type_name].update(self.frontend_node(name).to_dict())
+            node = self.frontend_node(name)
+            if node is not None:
+                node = node.to_dict()
+                result[self.type_name].update(node)
 
         return result
 
-    def frontend_node(self, name) -> FrontendNode:
+    def frontend_node(self, name) -> Union[FrontendNode, None]:
         signature = self.get_signature(name)
         if signature is None:
-            raise ValueError(f"{name} not found")
+            logger.error(f"Node {name} not loaded")
+            return None
         if isinstance(signature, FrontendNode):
             return signature
         fields = [

src/backend/langflow/interface/chains/base.py

@@ -5,6 +5,7 @@ from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import chain_type_to_cls_dict
 from langflow.settings import settings
 from langflow.template.nodes import ChainFrontendNode
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
@@ -39,6 +40,9 @@ class ChainCreator(LangChainTypeCreator):
             return build_template_from_class(name, self.type_to_loader_dict)
         except ValueError as exc:
             raise ValueError("Chain not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Chain {name} not loaded: {exc}")
+            return None
 
     def to_list(self) -> List[str]:
         custom_chains = list(get_custom_nodes("chains").keys())

src/backend/langflow/interface/custom_lists.py

@@ -1,6 +1,6 @@
+import inspect
 from typing import Any
 
-## LLM
 from langchain import (
     chains,
     document_loaders,
@@ -8,6 +8,7 @@ from langchain import (
     llms,
     memory,
     requests,
+    text_splitter,
     vectorstores,
 )
 from langchain.agents import agent_toolkits
@@ -15,16 +16,17 @@ from langchain.chat_models import ChatOpenAI
 
 from langflow.interface.importing.utils import import_class
 
-## LLM
+## LLMs
 llm_type_to_cls_dict = llms.type_to_cls_dict
 llm_type_to_cls_dict["openai-chat"] = ChatOpenAI  # type: ignore
 
-## Chain
+## Chains
 chain_type_to_cls_dict: dict[str, Any] = {
     chain_name: import_class(f"langchain.chains.{chain_name}")
     for chain_name in chains.__all__
 }
 
+## Toolkits
 toolkit_type_to_loader_dict: dict[str, Any] = {
     toolkit_name: import_class(f"langchain.agents.agent_toolkits.{toolkit_name}")
     # if toolkit_name is lower case it is a loader
@@ -69,3 +71,8 @@ documentloaders_type_to_cls_dict: dict[str, Any] = {
     )
     for documentloader_name in document_loaders.__all__
 }
+
+## Text Splitters
+textsplitter_type_to_cls_dict: dict[str, Any] = dict(
+    inspect.getmembers(text_splitter, inspect.isclass)
+)

src/backend/langflow/interface/documentLoaders/base.py

@@ -1,31 +0,0 @@
-from typing import Dict, List, Optional
-
-from langflow.interface.base import LangChainTypeCreator
-from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
-from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
-
-
-class DocumentLoaderCreator(LangChainTypeCreator):
-    type_name: str = "documentloader"
-
-    @property
-    def type_to_loader_dict(self) -> Dict:
-        return documentloaders_type_to_cls_dict
-
-    def get_signature(self, name: str) -> Optional[Dict]:
-        """Get the signature of a document loader."""
-        try:
-            return build_template_from_class(name, documentloaders_type_to_cls_dict)
-        except ValueError as exc:
-            raise ValueError(f"Documment Loader {name} not found") from exc
-
-    def to_list(self) -> List[str]:
-        return [
-            documentloader.__name__
-            for documentloader in self.type_to_loader_dict.values()
-            if documentloader.__name__ in settings.documentloaders or settings.dev
-        ]
-
-
-documentloader_creator = DocumentLoaderCreator()

src/backend/langflow/interface/document_loaders/base.py

@@ -0,0 +1,146 @@
+from typing import Dict, List, Optional
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
+
+
+def build_file_path_template(
+    suffixes: list, fileTypes: list, name: str = "file_path"
+) -> Dict:
+    """Build a file path template for a document loader."""
+    return {
+        "type": "file",
+        "required": True,
+        "show": True,
+        "name": name,
+        "value": "",
+        "suffixes": suffixes,
+        "fileTypes": fileTypes,
+    }
+
+
+class DocumentLoaderCreator(LangChainTypeCreator):
+    type_name: str = "documentloaders"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return documentloaders_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of a document loader."""
+        try:
+            signature = build_template_from_class(
+                name, documentloaders_type_to_cls_dict
+            )
+
+            file_path_templates = {
+                "AirbyteJSONLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "CoNLLULoader": build_file_path_template(
+                    suffixes=[".csv"], fileTypes=["csv"]
+                ),
+                "CSVLoader": build_file_path_template(
+                    suffixes=[".csv"], fileTypes=["csv"]
+                ),
+                "UnstructuredEmailLoader": build_file_path_template(
+                    suffixes=[".eml"], fileTypes=["eml"]
+                ),
+                "EverNoteLoader": build_file_path_template(
+                    suffixes=[".xml"], fileTypes=["xml"]
+                ),
+                "FacebookChatLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "GutenbergLoader": build_file_path_template(
+                    suffixes=[".txt"], fileTypes=["txt"]
+                ),
+                "BSHTMLLoader": build_file_path_template(
+                    suffixes=[".html"], fileTypes=["html"]
+                ),
+                "UnstructuredHTMLLoader": build_file_path_template(
+                    suffixes=[".html"], fileTypes=["html"]
+                ),
+                "UnstructuredImageLoader": build_file_path_template(
+                    suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
+                    fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
+                ),
+                "UnstructuredMarkdownLoader": build_file_path_template(
+                    suffixes=[".md"], fileTypes=["md"]
+                ),
+                "PyPDFLoader": build_file_path_template(
+                    suffixes=[".pdf"], fileTypes=["pdf"]
+                ),
+                "UnstructuredPowerPointLoader": build_file_path_template(
+                    suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
+                ),
+                "SRTLoader": build_file_path_template(
+                    suffixes=[".srt"], fileTypes=["srt"]
+                ),
+                "TelegramChatLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "TextLoader": build_file_path_template(
+                    suffixes=[".txt"], fileTypes=["txt"]
+                ),
+                "UnstructuredWordDocumentLoader": build_file_path_template(
+                    suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
+                ),
+            }
+
+            if name in file_path_templates:
+                signature["template"]["file_path"] = file_path_templates[name]
+            elif name in {
+                "WebBaseLoader",
+                "AZLyricsLoader",
+                "CollegeConfidentialLoader",
+                "HNLoader",
+                "IFixitLoader",
+                "IMSDbLoader",
+            }:
+                signature["template"]["web_path"] = {
+                    "type": "str",
+                    "required": True,
+                    "show": True,
+                    "name": "web_path",
+                    "value": "",
+                    "display_name": "Web Page",
+                }
+            elif name in {"GitbookLoader"}:
+                signature["template"]["web_page"] = {
+                    "type": "str",
+                    "required": True,
+                    "show": True,
+                    "name": "web_page",
+                    "value": "",
+                    "display_name": "Web Page",
+                }
+            elif name in {"ReadTheDocsLoader"}:
+                signature["template"]["path"] = {
+                    "type": "str",
+                    "required": True,
+                    "show": True,
+                    "name": "path",
+                    "value": "",
+                    "display_name": "Web Page",
+                }
+
+            return signature
+        except ValueError as exc:
+            raise ValueError(f"Documment Loader {name} not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Documment Loader {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            documentloader.__name__
+            for documentloader in self.type_to_loader_dict.values()
+            if documentloader.__name__ in settings.documentloaders or settings.dev
+        ]
+
+
+documentloader_creator = DocumentLoaderCreator()

src/backend/langflow/interface/embeddings/base.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Optional
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import embedding_type_to_cls_dict
 from langflow.settings import settings
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -20,6 +21,10 @@ class EmbeddingCreator(LangChainTypeCreator):
         except ValueError as exc:
             raise ValueError(f"Embedding {name} not found") from exc
 
+        except AttributeError as exc:
+            logger.error(f"Embedding {name} not loaded: {exc}")
+            return None
+
     def to_list(self) -> List[str]:
         return [
             embedding.__name__

src/backend/langflow/interface/importing/utils.py

@@ -40,6 +40,10 @@ def import_by_type(_type: str, name: str) -> Any:
         "toolkits": import_toolkit,
         "wrappers": import_wrapper,
         "memory": import_memory,
+        "embeddings": import_embedding,
+        "vectorstores": import_vectorstore,
+        "documentloaders": import_documentloader,
+        "textsplitters": import_textsplitter,
     }
     if _type == "llms":
         key = "chat" if "chat" in name.lower() else "llm"
@@ -113,3 +117,24 @@ def import_chain(chain: str) -> Type[Chain]:
     if chain in CUSTOM_CHAINS:
         return CUSTOM_CHAINS[chain]
     return import_class(f"langchain.chains.{chain}")
+
+
+def import_embedding(embedding: str) -> Any:
+    """Import embedding from embedding name"""
+    return import_class(f"langchain.embeddings.{embedding}")
+
+
+def import_vectorstore(vectorstore: str) -> Any:
+    """Import vectorstore from vectorstore name"""
+    return import_class(f"langchain.vectorstores.{vectorstore}")
+
+
+def import_documentloader(documentloader: str) -> Any:
+    """Import documentloader from documentloader name"""
+
+    return import_class(f"langchain.document_loaders.{documentloader}")
+
+
+def import_textsplitter(textsplitter: str) -> Any:
+    """Import textsplitter from textsplitter name"""
+    return import_class(f"langchain.text_splitter.{textsplitter}")

src/backend/langflow/interface/listing.py

@@ -1,10 +1,14 @@
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
+from langflow.interface.document_loaders.base import documentloader_creator
+from langflow.interface.embeddings.base import embedding_creator
 from langflow.interface.llms.base import llm_creator
 from langflow.interface.memories.base import memory_creator
 from langflow.interface.prompts.base import prompt_creator
+from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
 
@@ -18,6 +22,10 @@ def get_type_dict():
         "memory": memory_creator.to_list(),
         "toolkits": toolkits_creator.to_list(),
         "wrappers": wrapper_creator.to_list(),
+        "documentLoaders": documentloader_creator.to_list(),
+        "vectorStore": vectorstore_creator.to_list(),
+        "embeddings": embedding_creator.to_list(),
+        "textSplitters": textsplitter_creator.to_list(),
     }
 
 

src/backend/langflow/interface/llms/base.py

@@ -1,14 +1,20 @@
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Type
 
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import llm_type_to_cls_dict
 from langflow.settings import settings
+from langflow.template.nodes import LLMFrontendNode
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
 class LLMCreator(LangChainTypeCreator):
     type_name: str = "llms"
 
+    @property
+    def frontend_node_class(self) -> Type[LLMFrontendNode]:
+        return LLMFrontendNode
+
     @property
     def type_to_loader_dict(self) -> Dict:
         if self.type_dict is None:
@@ -22,6 +28,10 @@ class LLMCreator(LangChainTypeCreator):
         except ValueError as exc:
             raise ValueError("LLM not found") from exc
 
+        except AttributeError as exc:
+            logger.error(f"LLM {name} not loaded: {exc}")
+            return None
+
     def to_list(self) -> List[str]:
         return [
             llm.__name__

src/backend/langflow/interface/loading.py

@@ -22,6 +22,7 @@ from langflow.interface.agents.custom import CUSTOM_AGENTS
 from langflow.interface.importing.utils import import_by_type
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.types import get_type_list
+from langflow.interface.utils import load_file_into_dict
 from langflow.utils import util, validate
 
 
@@ -36,29 +37,44 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
     if base_type == "agents":
         # We need to initialize it differently
         return load_agent_executor(class_object, params)
-    elif node_type == "ZeroShotPrompt":
+    elif base_type == "prompts":
-        if "tools" not in params:
+        if node_type == "ZeroShotPrompt":
-            params["tools"] = []
+            if "tools" not in params:
-        return ZeroShotAgent.create_prompt(**params)
+                params["tools"] = []
-
+            return ZeroShotAgent.create_prompt(**params)
-    elif node_type == "PythonFunction":
+    elif base_type == "tools":
-        # If the node_type is "PythonFunction"
+        if node_type == "JsonSpec":
-        # we need to get the function from the params
+            params["dict_"] = load_file_into_dict(params.pop("path"))
-        # which will be a str containing a python function
+            return class_object(**params)
-        # and then we need to compile it and return the function
+        elif node_type == "PythonFunction":
-        # as the instance
+            # If the node_type is "PythonFunction"
-        function_string = params["code"]
+            # we need to get the function from the params
-        if isinstance(function_string, str):
+            # which will be a str containing a python function
-            return validate.eval_function(function_string)
+            # and then we need to compile it and return the function
-        raise ValueError("Function should be a string")
+            # as the instance
+            function_string = params["code"]
+            if isinstance(function_string, str):
+                return validate.eval_function(function_string)
+            raise ValueError("Function should be a string")
     elif base_type == "toolkits":
         loaded_toolkit = class_object(**params)
         # Check if node_type has a loader
         if toolkits_creator.has_create_function(node_type):
             return load_toolkits_executor(node_type, loaded_toolkit, params)
         return loaded_toolkit
-    else:
+    elif base_type == "embeddings":
+        params.pop("model")
         return class_object(**params)
+    elif base_type == "vectorstores":
+        return class_object.from_documents(**params)
+    elif base_type == "documentloaders":
+        return class_object(**params).load()
+    elif base_type == "textsplitters":
+        documents = params.pop("documents")
+        text_splitter = class_object(**params)
+        return text_splitter.split_documents(documents)
+
+    return class_object(**params)
 
 
 def load_flow_from_json(path: str):

src/backend/langflow/interface/memories/base.py

@@ -5,6 +5,7 @@ from langflow.interface.custom_lists import memory_type_to_cls_dict
 from langflow.settings import settings
 from langflow.template.base import FrontendNode
 from langflow.template.nodes import MemoryFrontendNode
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -28,6 +29,9 @@ class MemoryCreator(LangChainTypeCreator):
             return build_template_from_class(name, memory_type_to_cls_dict)
         except ValueError as exc:
             raise ValueError("Memory not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Memory {name} not loaded: {exc}")
+            return None
 
     def to_list(self) -> List[str]:
         return [

src/backend/langflow/interface/prompts/base.py

@@ -7,6 +7,7 @@ from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.importing.utils import import_class
 from langflow.settings import settings
 from langflow.template.nodes import PromptFrontendNode
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -43,7 +44,11 @@ class PromptCreator(LangChainTypeCreator):
                 return get_custom_nodes(self.type_name)[name]
             return build_template_from_class(name, self.type_to_loader_dict)
         except ValueError as exc:
-            raise ValueError("Prompt not found") from exc
+            # raise ValueError("Prompt not found") from exc
+            logger.error(f"Prompt {name} not found: {exc}")
+        except AttributeError as exc:
+            logger.error(f"Prompt {name} not loaded: {exc}")
+        return None
 
     def to_list(self) -> List[str]:
         custom_prompts = get_custom_nodes("prompts")

src/backend/langflow/interface/run.py

@@ -2,7 +2,7 @@ import contextlib
 import io
 from typing import Any, Dict
 
-from langflow.cache.utils import compute_hash, load_cache, save_cache
+from langflow.cache.utils import compute_dict_hash, load_cache, memoize_dict
 from langflow.graph.graph import Graph
 from langflow.interface import loading
 from langflow.utils.logger import logger
@@ -12,7 +12,7 @@ def load_langchain_object(data_graph, is_first_message=False):
     """
     Load langchain object from cache if it exists, otherwise build it.
     """
-    computed_hash = compute_hash(data_graph)
+    computed_hash = compute_dict_hash(data_graph)
     if is_first_message:
         langchain_object = build_langchain_object(data_graph)
     else:
@@ -22,6 +22,32 @@ def load_langchain_object(data_graph, is_first_message=False):
     return computed_hash, langchain_object
 
 
+def load_or_build_langchain_object(data_graph, is_first_message=False):
+    """
+    Load langchain object from cache if it exists, otherwise build it.
+    """
+    if is_first_message:
+        build_langchain_object_with_caching.clear_cache()
+    return build_langchain_object_with_caching(data_graph)
+
+
+@memoize_dict(maxsize=1)
+def build_langchain_object_with_caching(data_graph):
+    """
+    Build langchain object from data_graph.
+    """
+
+    logger.debug("Building langchain object")
+    nodes = data_graph["nodes"]
+    # Add input variables
+    # nodes = payload.extract_input_variables(nodes)
+    # Nodes, edges and root node
+    edges = data_graph["edges"]
+    graph = Graph(nodes, edges)
+
+    return graph.build()
+
+
 def build_langchain_object(data_graph):
     """
     Build langchain object from data_graph.
@@ -67,11 +93,35 @@ def process_graph(data_graph: Dict[str, Any]):
     # We have to save it here because if the
     # memory is updated we need to keep the new values
     logger.debug("Saving langchain object to cache")
-    save_cache(computed_hash, langchain_object, is_first_message)
+    # save_cache(computed_hash, langchain_object, is_first_message)
     logger.debug("Saved langchain object to cache")
     return {"result": str(result), "thought": thought.strip()}
 
 
+def process_graph_cached(data_graph: Dict[str, Any]):
+    """
+    Process graph by extracting input variables and replacing ZeroShotPrompt
+    with PromptTemplate,then run the graph and return the result and thought.
+    """
+    # Load langchain object
+    message = data_graph.pop("message", "")
+    is_first_message = len(data_graph.get("chatHistory", [])) == 0
+    langchain_object = load_or_build_langchain_object(data_graph, is_first_message)
+    logger.debug("Loaded langchain object")
+
+    if langchain_object is None:
+        # Raise user facing error
+        raise ValueError(
+            "There was an error loading the langchain_object. Please, check all the nodes and try again."
+        )
+
+    # Generate result and thought
+    logger.debug("Generating result and thought")
+    result, thought = get_result_and_thought_using_graph(langchain_object, message)
+    logger.debug("Generated result and thought")
+    return {"result": str(result), "thought": thought.strip()}
+
+
 def get_memory_key(langchain_object):
     """
     Given a LangChain object, this function retrieves the current memory key from the object's memory attribute.

src/backend/langflow/interface/text_splitters/__init__.py

@@ -0,0 +1,3 @@
+from langflow.interface.text_splitters.base import TextSplitterCreator
+
+__all__ = ["TextSplitterCreator"]

src/backend/langflow/interface/text_splitters/base.py

@@ -0,0 +1,71 @@
+from typing import Dict, List, Optional
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import textsplitter_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
+
+
+class TextSplitterCreator(LangChainTypeCreator):
+    type_name: str = "textsplitters"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return textsplitter_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of a text splitter."""
+        try:
+            signature = build_template_from_class(name, textsplitter_type_to_cls_dict)
+
+            signature["template"]["documents"] = {
+                "type": "BaseLoader",
+                "required": True,
+                "show": True,
+                "name": "documents",
+            }
+
+            signature["template"]["separator"] = {
+                "type": "str",
+                "required": True,
+                "show": True,
+                "value": ".",
+                "name": "separator",
+                "display_name": "Separator",
+            }
+
+            signature["template"]["chunk_size"] = {
+                "type": "int",
+                "required": True,
+                "show": True,
+                "value": 4000,
+                "name": "chunk_size",
+                "display_name": "Chunk Size",
+            }
+
+            signature["template"]["chunk_overlap"] = {
+                "type": "int",
+                "required": True,
+                "show": True,
+                "value": 200,
+                "name": "chunk_overlap",
+                "display_name": "Chunk Overlap",
+            }
+
+            return signature
+        except ValueError as exc:
+            raise ValueError(f"Text Splitter {name} not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Text Splitter {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            textsplitter.__name__
+            for textsplitter in self.type_to_loader_dict.values()
+            if textsplitter.__name__ in settings.textsplitters or settings.dev
+        ]
+
+
+textsplitter_creator = TextSplitterCreator()

src/backend/langflow/interface/toolkits/base.py

@@ -5,6 +5,7 @@ from langchain.agents import agent_toolkits
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.importing.utils import import_class, import_module
 from langflow.settings import settings
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -44,6 +45,9 @@ class ToolkitCreator(LangChainTypeCreator):
             return build_template_from_class(name, self.type_to_loader_dict)
         except ValueError as exc:
             raise ValueError("Prompt not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Prompt {name} not loaded: {exc}")
+            return None
 
     def to_list(self) -> List[str]:
         return list(self.type_to_loader_dict.keys())

src/backend/langflow/interface/tools/base.py

@@ -47,12 +47,14 @@ TOOL_INPUTS = {
         value="",
         multiline=True,
     ),
-    "dict_": TemplateField(
+    "path": TemplateField(
         field_type="file",
         required=True,
         is_list=False,
         show=True,
         value="",
+        suffixes=[".json", ".yaml", ".yml"],
+        fileTypes=["json", "yaml", "yml"],
     ),
 }
 
@@ -114,6 +116,8 @@ class ToolCreator(LangChainTypeCreator):
                 return node
         elif tool_type in FILE_TOOLS:
             params = all_tools[name]["params"]  # type: ignore
+            if tool_type == "JsonSpec":
+                params["path"] = params.pop("dict_")  # type: ignore
             base_classes += [name]
         else:
             params = []

src/backend/langflow/interface/tools/constants.py

@@ -7,7 +7,7 @@ from langchain.agents.load_tools import (
 )
 from langchain.tools.json.tool import JsonSpec
 
-from langflow.interface.custom.types import PythonFunction
+from langflow.interface.tools.custom import PythonFunction
 
 FILE_TOOLS = {"JsonSpec": JsonSpec}
 CUSTOM_TOOLS = {"Tool": Tool, "PythonFunction": PythonFunction}

src/backend/langflow/interface/types.py

@@ -1,13 +1,14 @@
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
-from langflow.interface.documentLoaders.base import documentloader_creator
+from langflow.interface.document_loaders.base import documentloader_creator
 from langflow.interface.embeddings.base import embedding_creator
 from langflow.interface.llms.base import llm_creator
 from langflow.interface.memories.base import memory_creator
 from langflow.interface.prompts.base import prompt_creator
+from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
-from langflow.interface.vectorStore.base import vectorstore_creator
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
 
@@ -40,6 +41,7 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
         embedding_creator,
         vectorstore_creator,
         documentloader_creator,
+        textsplitter_creator,
     ]
 
     all_types = {}

src/backend/langflow/interface/utils.py

@@ -0,0 +1,22 @@
+import json
+import os
+
+import yaml
+
+
+def load_file_into_dict(file_path: str) -> dict:
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    file_extension = os.path.splitext(file_path)[1].lower()
+
+    if file_extension == ".json":
+        with open(file_path, "r") as json_file:
+            data = json.load(json_file)
+    elif file_extension in [".yaml", ".yml"]:
+        with open(file_path, "r") as yaml_file:
+            data = yaml.safe_load(yaml_file)
+    else:
+        raise ValueError("Unsupported file type. Please provide a JSON or YAML file.")
+
+    return data

src/backend/langflow/interface/vectorStore/base.py

@@ -1,31 +0,0 @@
-from typing import Dict, List, Optional
-
-from langflow.interface.base import LangChainTypeCreator
-from langflow.interface.custom_lists import vectorstores_type_to_cls_dict
-from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
-
-
-class VectorstoreCreator(LangChainTypeCreator):
-    type_name: str = "vectorstore"
-
-    @property
-    def type_to_loader_dict(self) -> Dict:
-        return vectorstores_type_to_cls_dict
-
-    def get_signature(self, name: str) -> Optional[Dict]:
-        """Get the signature of an embedding."""
-        try:
-            return build_template_from_class(name, vectorstores_type_to_cls_dict)
-        except ValueError as exc:
-            raise ValueError(f"Vector Store {name} not found") from exc
-
-    def to_list(self) -> List[str]:
-        return [
-            vectorstore
-            for vectorstore in self.type_to_loader_dict.keys()
-            if vectorstore in settings.vectorstores or settings.dev
-        ]
-
-
-vectorstore_creator = VectorstoreCreator()

src/backend/langflow/interface/vector_store/base.py

@@ -0,0 +1,55 @@
+from typing import Dict, List, Optional
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import vectorstores_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
+
+
+class VectorstoreCreator(LangChainTypeCreator):
+    type_name: str = "vectorstores"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return vectorstores_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of an embedding."""
+        try:
+            signature = build_template_from_class(name, vectorstores_type_to_cls_dict)
+
+            # TODO: Use FrontendendNode class to build the signature
+            signature["template"] = {
+                "documents": {
+                    "type": "TextSplitter",
+                    "required": True,
+                    "show": True,
+                    "name": "documents",
+                    "display_name": "Text Splitter",
+                },
+                "embedding": {
+                    "type": "Embeddings",
+                    "required": True,
+                    "show": True,
+                    "name": "embedding",
+                    "display_name": "Embedding",
+                },
+            }
+            return signature
+
+        except ValueError as exc:
+            raise ValueError(f"Vector Store {name} not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Vector Store {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            vectorstore
+            for vectorstore in self.type_to_loader_dict.keys()
+            if vectorstore in settings.vectorstores or settings.dev
+        ]
+
+
+vectorstore_creator = VectorstoreCreator()

src/backend/langflow/interface/wrappers/base.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Optional
 from langchain import requests
 
 from langflow.interface.base import LangChainTypeCreator
+from langflow.utils.logger import logger
 from langflow.utils.util import build_template_from_class
 
 
@@ -22,6 +23,9 @@ class WrapperCreator(LangChainTypeCreator):
             return build_template_from_class(name, self.type_to_loader_dict)
         except ValueError as exc:
             raise ValueError("Wrapper not found") from exc
+        except AttributeError as exc:
+            logger.error(f"Wrapper {name} not loaded: {exc}")
+            return None
 
     def to_list(self) -> List[str]:
         return list(self.type_to_loader_dict.keys())

src/backend/langflow/settings.py

@@ -17,6 +17,7 @@ class Settings(BaseSettings):
     documentloaders: List[str] = []
     wrappers: List[str] = []
     toolkits: List[str] = []
+    textsplitters: List[str] = []
     dev: bool = False
 
     class Config:
@@ -40,6 +41,7 @@ class Settings(BaseSettings):
         self.memories = new_settings.memories or []
         self.wrappers = new_settings.wrappers or []
         self.toolkits = new_settings.toolkits or []
+        self.textsplitters = new_settings.textsplitters or []
         self.dev = new_settings.dev or False
 
 

src/backend/langflow/template/nodes.py

@@ -248,6 +248,93 @@ class CSVAgentNode(FrontendNode):
         return super().to_dict()
 
 
+class VectorStoreAgentNode(FrontendNode):
+    name: str = "VectorStoreAgent"
+    template: Template = Template(
+        type_name="vectorstore_agent",
+        fields=[
+            TemplateField(
+                field_type="VectorStoreInfo",
+                required=True,
+                show=True,
+                name="vectorstoreinfo",
+                display_name="Vector Store Info",
+            ),
+            TemplateField(
+                field_type="BaseLanguageModel",
+                required=True,
+                show=True,
+                name="llm",
+                display_name="LLM",
+            ),
+        ],
+    )
+    description: str = """Construct an agent from a Vector Store."""
+    base_classes: list[str] = ["AgentExecutor"]
+
+    def to_dict(self):
+        return super().to_dict()
+
+
+class VectorStoreRouterAgentNode(FrontendNode):
+    name: str = "VectorStoreRouterAgent"
+    template: Template = Template(
+        type_name="vectorstorerouter_agent",
+        fields=[
+            TemplateField(
+                field_type="VectorStoreRouterToolkit",
+                required=True,
+                show=True,
+                name="vectorstoreroutertoolkit",
+                display_name="Vector Store Router Toolkit",
+            ),
+            TemplateField(
+                field_type="BaseLanguageModel",
+                required=True,
+                show=True,
+                name="llm",
+                display_name="LLM",
+            ),
+        ],
+    )
+    description: str = """Construct an agent from a Vector Store Router."""
+    base_classes: list[str] = ["AgentExecutor"]
+
+    def to_dict(self):
+        return super().to_dict()
+
+
+class SQLAgentNode(FrontendNode):
+    name: str = "SQLAgent"
+    template: Template = Template(
+        type_name="sql_agent",
+        fields=[
+            TemplateField(
+                field_type="str",
+                required=True,
+                placeholder="",
+                is_list=False,
+                show=True,
+                multiline=False,
+                value="",
+                name="database_uri",
+            ),
+            TemplateField(
+                field_type="BaseLanguageModel",
+                required=True,
+                show=True,
+                name="llm",
+                display_name="LLM",
+            ),
+        ],
+    )
+    description: str = """Construct an agent from a Vector Store Router."""
+    base_classes: list[str] = ["AgentExecutor"]
+
+    def to_dict(self):
+        return super().to_dict()
+
+
 class PromptFrontendNode(FrontendNode):
     @staticmethod
     def format_field(field: TemplateField, name: Optional[str] = None) -> None:
@@ -275,6 +362,9 @@ class PromptFrontendNode(FrontendNode):
         ):
             field.field_type = "BaseMessagePromptTemplate"
 
+        # All prompt fields should be password=False
+        field.password = False
+
 
 class MemoryFrontendNode(FrontendNode):
     @staticmethod
@@ -289,6 +379,7 @@ class MemoryFrontendNode(FrontendNode):
             field.field_type = "int"
             field.value = 10
             field.display_name = "Memory Size"
+        field.password = False
 
 
 class ChainFrontendNode(FrontendNode):
@@ -299,3 +390,39 @@ class ChainFrontendNode(FrontendNode):
         if "key" in field.name:
             field.password = False
             field.show = False
+        if field.name in ["input_key", "output_key"]:
+            field.required = True
+            field.show = True
+        # Separated for possible future changes
+        if field.name == "prompt":
+            # if no prompt is provided, use the default prompt
+            field.required = False
+            field.show = True
+
+
+class LLMFrontendNode(FrontendNode):
+    @staticmethod
+    def format_field(field: TemplateField, name: Optional[str] = None) -> None:
+        display_names_dict = {
+            "huggingfacehub_api_token": "HuggingFace Hub API Token",
+        }
+        FrontendNode.format_field(field, name)
+        SHOW_FIELDS = ["repo_id", "task", "model_kwargs"]
+        if field.name in SHOW_FIELDS:
+            field.show = True
+
+        if "api" in field.name and ("key" in field.name or "token" in field.name):
+            field.password = True
+            field.show = True
+            field.required = True
+
+        if field.name == "task":
+            field.required = True
+            field.show = True
+            field.is_list = True
+            field.options = ["text-generation", "text2text-generation"]
+
+        if display_name := display_names_dict.get(field.name):
+            field.display_name = display_name
+        if field.name == "model_kwargs":
+            field.field_type = "code"

src/backend/langflow/utils/util.py

@@ -3,6 +3,8 @@ import inspect
 import re
 from typing import Dict, Optional
 
+from docstring_parser import parse  # type: ignore
+
 from langflow.template.constants import FORCE_SHOW_FIELDS
 from langflow.utils import constants
 
@@ -65,7 +67,8 @@ def build_template_from_function(
         if v.__annotations__["return"].__name__ == name:
             _class = v.__annotations__["return"]
 
-            docs = get_class_doc(_class)
+            # Get the docstring
+            docs = parse(_class.__doc__)
 
             variables = {"_type": _type}
             for class_field_items, value in _class.__fields__.items():
@@ -86,8 +89,8 @@ def build_template_from_function(
                         variables[class_field_items][name_] = value_
 
                 variables[class_field_items]["placeholder"] = (
-                    docs["Attributes"][class_field_items]
+                    docs.params[class_field_items]
-                    if class_field_items in docs["Attributes"]
+                    if class_field_items in docs.params
                     else ""
                 )
             # Adding function to base classes to allow
@@ -98,7 +101,7 @@ def build_template_from_function(
 
             return {
                 "template": format_dict(variables, name),
-                "description": docs["Description"],
+                "description": docs.short_description or "",
                 "base_classes": base_classes,
             }
 
@@ -117,7 +120,7 @@ def build_template_from_class(
             _class = v
 
             # Get the docstring
-            docs = get_class_doc(_class)
+            docs = parse(_class.__doc__)
 
             variables = {"_type": _type}
 
@@ -140,8 +143,8 @@ def build_template_from_class(
                             variables[class_field_items][name_] = value_
 
                     variables[class_field_items]["placeholder"] = (
-                        docs["Attributes"][class_field_items]
+                        docs.params[class_field_items]
-                        if class_field_items in docs["Attributes"]
+                        if class_field_items in docs.params
                         else ""
                     )
             base_classes = get_base_classes(_class)
@@ -151,7 +154,7 @@ def build_template_from_class(
                 base_classes.append("function")
             return {
                 "template": format_dict(variables, name),
-                "description": docs["Description"],
+                "description": docs.short_description or "",
                 "base_classes": base_classes,
             }
 
@@ -188,60 +191,6 @@ def get_default_factory(module: str, function: str):
     return None
 
 
-def get_class_doc(class_name):
-    """
-    Extracts information from the docstring of a given class.
-
-    Args:
-        class_name: the class to extract information from
-
-    Returns:
-        A dictionary containing the extracted information, with keys
-        for 'Description', 'Parameters', 'Attributes', and 'Returns'.
-    """
-    # Template
-    data = {
-        "Description": "",
-        "Parameters": {},
-        "Attributes": {},
-        "Example": [],
-        "Returns": {},
-    }
-
-    # Get the class docstring
-    docstring = class_name.__doc__
-
-    if not docstring:
-        return data
-
-    # Parse the docstring to extract information
-    lines = docstring.split("\n")
-
-    current_section = "Description"
-
-    for line in lines:
-        line = line.strip()
-
-        if not line:
-            continue
-
-        if (
-            line.startswith(tuple(data.keys()))
-            and len(line.split()) == 1
-            and line.endswith(":")
-        ):
-            current_section = line[:-1]
-            continue
-
-        if current_section in ["Description", "Example"]:
-            data[current_section] += line
-        else:
-            param, desc = line.split(":")
-            data[current_section][param.strip()] = desc.strip()
-
-    return data
-
-
 def format_dict(d, name: Optional[str] = None):
     """
     Formats a dictionary by removing certain keys and modifying the

src/frontend/src/App.css

@@ -40,3 +40,8 @@
     transform: rotate(360deg);
   }
 }
+
+@font-face{
+  font-family: text-security-disc;
+  src: url("assets/text-security-disc.woff") format("woff");
+}

src/frontend/src/components/inputComponent/index.tsx

@@ -23,7 +23,7 @@ export default function InputComponent({
 				className={classNames(
 					"block w-full form-input dark:bg-gray-900 dark:border-gray-600 rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 sm:text-sm",
 					disabled ? " bg-gray-200 dark:bg-gray-700" : "",
-                    password?"password":""
+                    password && myValue.length>0?"password":""
 				)}
 				placeholder="Type a text"
 				onChange={(e) => {

src/frontend/src/pages/FlowPage/index.tsx

@@ -6,9 +6,10 @@ import ReactFlow, {
 	useEdgesState,
 	useNodesState,
 	useReactFlow,
-	ControlButton,
+	updateEdge,
 	EdgeChange,
 	Connection,
+	Edge,
 } from "reactflow";
 import { locationContext } from "../../contexts/locationContext";
 import ExtraSidebar from "./components/extraSidebarComponent";
@@ -20,6 +21,7 @@ import { typesContext } from "../../contexts/typesContext";
 import ConnectionLineComponent from "./components/ConnectionLineComponent";
 import { FlowType, NodeType } from "../../types/flow";
 import { APIClassType } from "../../types/api";
+import { isValidConnection } from "../../utils";
 
 const nodeTypes = {
 	genericNode: GenericNode,
@@ -28,7 +30,7 @@ const nodeTypes = {
 var _ = require("lodash");
 
 export default function FlowPage({ flow }:{flow:FlowType}) {
-	let { updateFlow, incrementNodeId, downloadFlow, uploadFlow } =
+	let { updateFlow, incrementNodeId} =
 		useContext(TabsContext);
 	const { types, reactFlowInstance, setReactFlowInstance } =
 		useContext(typesContext);
@@ -43,6 +45,7 @@ export default function FlowPage({ flow }:{flow:FlowType}) {
 		flow.data?.edges ?? []
 	);
 	const { setViewport } = useReactFlow();
+	const edgeUpdateSuccessful = useRef(true)
 
 	useEffect(() => {
 		if (reactFlowInstance && flow) {
@@ -155,6 +158,26 @@ export default function FlowPage({ flow }:{flow:FlowType}) {
 		setEdges(edges.filter((ns) => !nodes.some((n) => ns.source === n.id || ns.target === n.id)));
 	}
 
+	const onEdgeUpdateStart = useCallback(() => {
+		edgeUpdateSuccessful.current = false;
+	  }, []);	
+
+	
+	  const onEdgeUpdate = useCallback((oldEdge:Edge, newConnection:Connection) => {
+		if(isValidConnection(newConnection,reactFlowInstance)){
+			edgeUpdateSuccessful.current = true;
+			setEdges((els) => updateEdge(oldEdge, newConnection, els));
+		}
+	}, []);
+
+	const onEdgeUpdateEnd = useCallback((_, edge) => {
+		if (!edgeUpdateSuccessful.current) {
+		  setEdges((eds) => eds.filter((e) => e.id !== edge.id));
+		}
+	
+		edgeUpdateSuccessful.current = true;
+	  }, []);
+	  
 	return (
 		<div className="w-full h-full" ref={reactFlowWrapper}>
 			{Object.keys(types).length > 0 ? (
@@ -171,6 +194,9 @@ export default function FlowPage({ flow }:{flow:FlowType}) {
 						onLoad={setReactFlowInstance}
 						onInit={setReactFlowInstance}
 						nodeTypes={nodeTypes}
+						onEdgeUpdate={onEdgeUpdate}
+						onEdgeUpdateStart={onEdgeUpdateStart}
+						onEdgeUpdateEnd={onEdgeUpdateEnd}
 						connectionLineComponent={ConnectionLineComponent}
 						onDragOver={onDragOver}
 						onDrop={onDrop}

src/frontend/src/utils.ts

@@ -9,7 +9,11 @@ import {
   ComputerDesktopIcon,
   Bars3CenterLeftIcon,
   GiftIcon,
+  PaperClipIcon,
   QuestionMarkCircleIcon,
+  FingerPrintIcon,
+  ScissorsIcon,
+  CircleStackIcon
 } from "@heroicons/react/24/outline";
 import { Connection, Edge, Node, ReactFlowInstance } from "reactflow";
 import { FlowType } from "./types/flow";
@@ -71,11 +75,14 @@ export const nodeColors: {[char: string]: string} = {
   chains: "#FE7500",
   agents: "#903BBE",
   tools: "#FF3434",
-  memories: "#FF9135",
+  memories: "#F5B85A",
   advanced: "#000000",
   chat: "#454173",
   thought:"#272541",
-  docloaders:"#FF9135",
+  embeddings:"#42BAA7",
+  documentloaders:"#7AAE42",
+  vectorstores: "#AA8742",
+  textsplitters: "#B47CB5",
   toolkits:"#DB2C2C",
   wrappers:"#E6277A",
   unknown:"#9CA3AF"
@@ -90,9 +97,12 @@ export const nodeNames:{[char: string]: string} = {
   memories: "Memories",
   advanced: "Advanced",
   chat: "Chat",
-  docloaders:"Document Loader",
+  embeddings: "Embeddings",
+  documentloaders: "Document Loaders",
+  vectorstores: "Vector Stores",
   toolkits:"Toolkits",
   wrappers:"Wrappers",
+  textsplitters: "Text Splitters",
   unknown:"Unknown"
 };
 
@@ -105,8 +115,11 @@ export const nodeIcons:{[char: string]: React.ForwardRefExoticComponent<React.SV
   tools: WrenchIcon,
   advanced: ComputerDesktopIcon,
   chat: Bars3CenterLeftIcon,
-  docloaders:Bars3CenterLeftIcon,
+  embeddings:FingerPrintIcon,
+  documentloaders:PaperClipIcon,
+  vectorstores: CircleStackIcon,
   toolkits:WrenchScrewdriverIcon,
+  textsplitters:ScissorsIcon,
   wrappers:GiftIcon,
   unknown:QuestionMarkCircleIcon
 };

src/frontend/tailwind.config.js

@@ -34,7 +34,9 @@ module.exports = {
           },
         },
         '.password':{
-          "-webkit-text-security":"disc"
+          "-webkit-text-security":"disc",
+          "font-family": "text-security-disc"
+
         }
       }
       )

tests/data/Openapi.json

@@ -206,7 +206,7 @@
                     "type": "JsonSpec",
                     "node": {
                         "template": {
-                            "dict_": {
+                            "path": {
                                 "required": true,
                                 "placeholder": "",
                                 "show": true,
@@ -218,7 +218,7 @@
                                     ".yml"
                                 ],
                                 "password": false,
-                                "name": "dict_",
+                                "name": "path",
                                 "type": "file",
                                 "list": false,
                                 "fileTypes": [

tests/test_cache.py

@@ -49,7 +49,7 @@ def test_cache_creation(basic_data_graph):
     )
     save_cache(computed_hash, langchain_object, is_first_message)
     # Check if the cache file exists
-    cache_file = Path(tempfile.gettempdir()) / f"{PREFIX}_{computed_hash}.dill"
+    cache_file = Path(tempfile.gettempdir()) / f"{PREFIX}/{computed_hash}.dill"
 
     assert cache_file.exists()
 

tests/test_chains_template.py

@@ -53,9 +53,9 @@ def test_conversation_chain(client: TestClient):
         "list": False,
     }
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "input",
         "password": False,
@@ -64,9 +64,9 @@ def test_conversation_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "response",
         "password": False,
@@ -125,9 +125,9 @@ def test_llm_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "text",
         "password": False,
@@ -179,9 +179,9 @@ def test_llm_checker_chain(client: TestClient):
         "list": False,
     }
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "query",
         "password": False,
@@ -190,9 +190,9 @@ def test_llm_checker_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "result",
         "password": False,
@@ -251,9 +251,9 @@ def test_llm_math_chain(client: TestClient):
         "list": False,
     }
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "question",
         "password": False,
@@ -262,9 +262,9 @@ def test_llm_math_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "answer",
         "password": False,
@@ -338,9 +338,9 @@ def test_series_character_chain(client: TestClient):
         "list": False,
     }
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "input",
         "password": False,
@@ -349,9 +349,9 @@ def test_series_character_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "response",
         "password": False,
@@ -462,7 +462,7 @@ def test_mid_journey_prompt_chain(client: TestClient):
     assert template["prompt"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": {
             "input_variables": ["history", "input"],
@@ -489,9 +489,9 @@ def test_mid_journey_prompt_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "response",
         "password": False,
@@ -500,9 +500,9 @@ def test_mid_journey_prompt_chain(client: TestClient):
         "list": False,
     }
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "input",
         "password": False,
@@ -590,7 +590,7 @@ def test_time_travel_guide_chain(client: TestClient):
     assert template["prompt"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": {
             "input_variables": ["history", "input"],
@@ -617,9 +617,9 @@ def test_time_travel_guide_chain(client: TestClient):
         "list": False,
     }
     assert template["output_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "response",
         "password": False,
@@ -629,9 +629,9 @@ def test_time_travel_guide_chain(client: TestClient):
     }
 
     assert template["input_key"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "input",
         "password": False,

tests/test_custom_types.py

@@ -1,6 +1,6 @@
 # Test this:
 import pytest
-from langflow.interface.custom.types import PythonFunction
+from langflow.interface.tools.custom import PythonFunction
 from langflow.utils import constants
 
 

tests/test_graph.py

@@ -1,7 +1,9 @@
 import json
+from typing import Type, Union
 
 import pytest
 from langchain.agents import AgentExecutor
+from langchain.llms.fake import FakeListLLM
 from langflow.graph import Edge, Graph, Node
 from langflow.graph.nodes import (
     AgentNode,
@@ -13,6 +15,7 @@ from langflow.graph.nodes import (
     ToolNode,
     WrapperNode,
 )
+from langflow.interface.run import get_result_and_thought_using_graph
 from langflow.utils.payload import build_json, get_root_node
 
 # Test cases for the graph module
@@ -53,7 +56,7 @@ def openapi_graph():
     return get_graph("openapi")
 
 
-def get_node_by_type(graph, node_type):
+def get_node_by_type(graph, node_type: Type[Node]) -> Union[Node, None]:
     """Get a node by type"""
     return next((node for node in graph.nodes if isinstance(node, node_type)), None)
 
@@ -418,3 +421,28 @@ def test_wrapper_node_build(openapi_graph):
     built_object = wrapper_node.build()
     assert built_object is not None
     # Add any further assertions specific to the WrapperNode's build() method
+
+
+def test_get_result_and_thought(basic_graph):
+    """Test the get_result_and_thought method"""
+    responses = [
+        "Final Answer: I am a response",
+    ]
+    message = "Hello"
+    # Find the node that is an LLMNode and change the
+    # _built_object to a FakeListLLM
+    llm_node = get_node_by_type(basic_graph, LLMNode)
+    assert llm_node is not None
+    llm_node._built_object = FakeListLLM(responses=responses)
+    llm_node._built = True
+    langchain_object = basic_graph.build()
+    # assert all nodes are built
+    assert all(node._built for node in basic_graph.nodes)
+    # now build again and check if FakeListLLM was used
+
+    # Get the result and thought
+    result, thought = get_result_and_thought_using_graph(langchain_object, message)
+    # The result should be a str
+    assert isinstance(result, str)
+    # The thought should be a Thought
+    assert isinstance(thought, str)

tests/test_llms_template.py

@@ -53,7 +53,7 @@ def test_hugging_face_hub(client: TestClient):
     assert template["repo_id"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "value": "gpt2",
         "password": False,
@@ -62,19 +62,20 @@ def test_hugging_face_hub(client: TestClient):
         "list": False,
     }
     assert template["task"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "password": False,
+        "options": ["text-generation", "text2text-generation"],
         "name": "task",
         "type": "str",
-        "list": False,
+        "list": True,
     }
     assert template["model_kwargs"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "password": False,
         "name": "model_kwargs",
@@ -82,12 +83,13 @@ def test_hugging_face_hub(client: TestClient):
         "list": False,
     }
     assert template["huggingfacehub_api_token"] == {
-        "required": False,
+        "required": True,
         "placeholder": "",
         "show": True,
         "multiline": False,
         "password": True,
         "name": "huggingfacehub_api_token",
+        "display_name": "HuggingFace Hub API Token",
         "type": "str",
         "list": False,
     }
@@ -231,7 +233,7 @@ def test_openai(client: TestClient):
     assert template["model_kwargs"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "password": False,
         "name": "model_kwargs",
@@ -361,7 +363,7 @@ def test_chat_open_ai(client: TestClient):
     assert template["model_kwargs"] == {
         "required": False,
         "placeholder": "",
-        "show": False,
+        "show": True,
         "multiline": False,
         "password": False,
         "name": "model_kwargs",
@@ -438,7 +440,7 @@ def test_chat_open_ai(client: TestClient):
     assert template["_type"] == "ChatOpenAI"
     assert (
         model["description"]
-        == "Wrapper around OpenAI Chat large language models.To use, you should have the ``openai`` python package installed, and theenvironment variable ``OPENAI_API_KEY`` set with your API key.Any parameters that are valid to be passed to the openai.create call can be passedin, even if not explicitly saved on this class."  # noqa E501
+        == "Wrapper around OpenAI Chat large language models."  # noqa E501
     )
     assert set(model["base_classes"]) == {
         "BaseChatModel",

tests/test_template.py

@@ -8,7 +8,6 @@ from langflow.utils.util import (
     build_template_from_function,
     format_dict,
     get_base_classes,
-    get_class_doc,
     get_default_factory,
 )
 from pydantic import BaseModel
@@ -280,12 +279,3 @@ def test_get_default_factory():
     default_value = get_default_factory(module_name, function_repr)
 
     assert default_value == "default_value"
-
-
-# Test get_class_doc
-def test_get_class_doc():
-    class_doc_parent = get_class_doc(Parent)
-    class_doc_child = get_class_doc(Child)
-
-    assert class_doc_parent["Description"] == "Parent Class"
-    assert class_doc_child["Description"] == "Child Class"

tests/test_vectorstore_template.py

@@ -0,0 +1,12 @@
+from fastapi.testclient import TestClient
+from langflow.settings import settings
+
+
+# check that all agents are in settings.agents
+# are in json_response["agents"]
+def test_vectorstores_settings(client: TestClient):
+    response = client.get("/all")
+    assert response.status_code == 200
+    json_response = response.json()
+    vectorstores = json_response["vectorstores"]
+    assert set(vectorstores.keys()) == set(settings.vectorstores)