Merge branch 'dev' into autoUpdateNodes

src/backend/langflow/config.yaml

@@ -6,6 +6,7 @@ chains:
   - SeriesCharacterChain
   - MidJourneyPromptChain
   - TimeTravelGuideChain
+  - SQLDatabaseChain
 
 agents:
   - ZeroShotAgent
@@ -40,6 +41,27 @@ tools:
   - Tool
   - PythonFunction
   - JsonSpec
+  - News API
+  - TMDB API
+  - Podcast API
+  - QuerySQLDataBaseTool
+  - InfoSQLDatabaseTool
+  - ListSQLDatabaseTool
+  # - QueryCheckerTool
+  - BingSearchRun
+  - GoogleSearchRun
+  - GoogleSearchResults
+  - JsonListKeysTool
+  - JsonGetValueTool
+  - PythonREPLTool
+  - PythonAstREPLTool
+  - RequestsGetTool
+  - RequestsPostTool
+  - RequestsPatchTool
+  - RequestsPutTool
+  - RequestsDeleteTool
+  - WikipediaQueryRun
+  - WolframAlphaQueryRun
 
 wrappers:
   - RequestsWrapper
@@ -91,4 +113,16 @@ documentloaders:
 textsplitters:
   - CharacterTextSplitter
 
+utilities:
+  - BingSearchAPIWrapper
+  - GoogleSearchAPIWrapper
+  - GoogleSerperAPIWrapper
+  - SearxResults
+  - SearxSearchWrapper
+  - SerpAPIWrapper
+  - WikipediaAPIWrapper
+  - WolframAlphaAPIWrapper
+  # - ZapierNLAWrapper
+  - SQLDatabase
+
 dev: false

src/backend/langflow/custom/customs.py

@@ -12,6 +12,9 @@ CUSTOM_NODES = {
         "VectorStoreRouterAgent": nodes.VectorStoreRouterAgentNode(),
         "SQLAgent": nodes.SQLAgentNode(),
     },
+    "utilities": {
+        "SQLDatabase": nodes.SQLDatabaseNode(),
+    },
 }
 
 

src/backend/langflow/graph/base.py

@@ -202,7 +202,11 @@ class Node:
             "VectorStoreRouterAgent",
             "VectorStoreAgent",
             "VectorStoreInfo",
-        ] or self.node_type in ["VectorStoreInfo", "VectorStoreRouterToolkit"]:
+        ] or self.node_type in [
+            "VectorStoreInfo",
+            "VectorStoreRouterToolkit",
+            "SQLDatabase",
+        ]:
             return self._built_object
         return deepcopy(self._built_object)
 

src/backend/langflow/graph/nodes.py

@@ -101,6 +101,10 @@ class ChainNode(Node):
                     self.params[key] = value.build(tools=tools, force=force)
 
             self._build()
+
+        #! Cannot deepcopy SQLDatabaseChain
+        if self.node_type in ["SQLDatabaseChain"]:
+            return self._built_object
         return deepcopy(self._built_object)
 
 

src/backend/langflow/interface/custom_lists.py

@@ -9,10 +9,12 @@ from langchain import (
     memory,
     requests,
     text_splitter,
+    utilities,
     vectorstores,
 )
 from langchain.agents import agent_toolkits
 from langchain.chat_models import ChatOpenAI
+from langchain.sql_database import SQLDatabase
 
 from langflow.interface.importing.utils import import_class
 
@@ -76,3 +78,9 @@ documentloaders_type_to_cls_dict: dict[str, Any] = {
 textsplitter_type_to_cls_dict: dict[str, Any] = dict(
     inspect.getmembers(text_splitter, inspect.isclass)
 )
+
+## Utilities
+utility_type_to_cls_dict: dict[str, Any] = dict(
+    inspect.getmembers(utilities, inspect.isclass)
+)
+utility_type_to_cls_dict["SQLDatabase"] = SQLDatabase

src/backend/langflow/interface/importing/utils.py

@@ -10,7 +10,7 @@ from langchain.chat_models.base import BaseChatModel
 from langchain.llms.base import BaseLLM
 from langchain.tools import BaseTool
 
-from langflow.interface.tools.util import get_tool_by_name
+from langflow.interface.tools.base import tool_creator
 
 
 def import_module(module_path: str) -> Any:
@@ -44,6 +44,7 @@ def import_by_type(_type: str, name: str) -> Any:
         "vectorstores": import_vectorstore,
         "documentloaders": import_documentloader,
         "textsplitters": import_textsplitter,
+        "utilities": import_utility,
     }
     if _type == "llms":
         key = "chat" if "chat" in name.lower() else "llm"
@@ -107,7 +108,7 @@ def import_llm(llm: str) -> BaseLLM:
 def import_tool(tool: str) -> BaseTool:
     """Import tool from tool name"""
 
-    return get_tool_by_name(tool)
+    return tool_creator.type_to_loader_dict[tool]["fcn"]
 
 
 def import_chain(chain: str) -> Type[Chain]:
@@ -131,10 +132,16 @@ def import_vectorstore(vectorstore: str) -> Any:
 
 def import_documentloader(documentloader: str) -> Any:
     """Import documentloader from documentloader name"""
-
     return import_class(f"langchain.document_loaders.{documentloader}")
 
 
 def import_textsplitter(textsplitter: str) -> Any:
     """Import textsplitter from textsplitter name"""
     return import_class(f"langchain.text_splitter.{textsplitter}")
+
+
+def import_utility(utility: str) -> Any:
+    """Import utility from utility name"""
+    if utility == "SQLDatabase":
+        return import_class(f"langchain.sql_database.{utility}")
+    return import_class(f"langchain.utilities.{utility}")

src/backend/langflow/interface/listing.py

@@ -8,6 +8,7 @@ from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
+from langflow.interface.utilities.base import utility_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
@@ -26,6 +27,7 @@ def get_type_dict():
         "vectorStore": vectorstore_creator.to_list(),
         "embeddings": embedding_creator.to_list(),
         "textSplitters": textsplitter_creator.to_list(),
+        "utilities": utility_creator.to_list(),
     }
 
 

src/backend/langflow/interface/loading.py

@@ -68,6 +68,13 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
         params.pop("model")
         return class_object(**params)
     elif base_type == "vectorstores":
+        if len(params.get("documents", [])) == 0:
+            # Error when the pdf or other source was not correctly
+            # loaded.
+            raise ValueError(
+                "The source you provided did not load correctly or was empty."
+                "This may cause an error in the vectorstore."
+            )
         return class_object.from_documents(**params)
     elif base_type == "documentloaders":
         return class_object(**params).load()
@@ -75,16 +82,19 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
         documents = params.pop("documents")
         text_splitter = class_object(**params)
         return text_splitter.split_documents(documents)
+    elif base_type == "utilities":
+        if node_type == "SQLDatabase":
+            return class_object.from_uri(params.pop("uri"))
 
     return class_object(**params)
 
 
-def load_flow_from_json(path: str):
+def load_flow_from_json(path: str, build=True):
     # This is done to avoid circular imports
     from langflow.graph import Graph
 
     """Load flow from json file"""
-    with open(path, "r") as f:
+    with open(path, "r", encoding="utf-8") as f:
         flow_graph = json.load(f)
     data_graph = flow_graph["data"]
     nodes = data_graph["nodes"]
@@ -96,7 +106,7 @@ def load_flow_from_json(path: str):
     # Nodes, edges and root node
     edges = data_graph["edges"]
     graph = Graph(nodes, edges)
-    return graph.build()
+    return graph.build() if build else graph
 
 
 def replace_zero_shot_prompt_with_prompt_template(nodes):

src/backend/langflow/interface/run.py

@@ -1,6 +1,7 @@
 import contextlib
 import io
 from typing import Any, Dict
+from chromadb.errors import NotEnoughElementsException
 
 from langflow.cache.utils import compute_dict_hash, load_cache, memoize_dict
 from langflow.graph.graph import Graph
@@ -230,6 +231,10 @@ def get_result_and_thought_using_graph(langchain_object, message: str):
             else:
                 thought = output_buffer.getvalue()
 
+    except NotEnoughElementsException as exc:
+        raise ValueError(
+            "Error: Not enough documents for ChromaDB to index. Try reducing chunk size in TextSplitter."
+        ) from exc
     except Exception as exc:
         raise ValueError(f"Error: {str(exc)}") from exc
     return result, thought

src/backend/langflow/interface/tools/base.py

@@ -1,7 +1,6 @@
 from typing import Dict, List, Optional
 
 from langchain.agents.load_tools import (
-    _BASE_TOOLS,
     _EXTRA_LLM_TOOLS,
     _EXTRA_OPTIONAL_TOOLS,
     _LLM_TOOLS,
@@ -10,17 +9,16 @@ from langchain.agents.load_tools import (
 from langflow.custom import customs
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.tools.constants import (
+    ALL_TOOLS_NAMES,
     CUSTOM_TOOLS,
     FILE_TOOLS,
+    OTHER_TOOLS,
 )
-from langflow.interface.tools.util import (
-    get_tool_by_name,
-    get_tool_params,
-    get_tools_dict,
-)
+from langflow.interface.tools.util import get_tool_params
 from langflow.settings import settings
 from langflow.template.base import Template, TemplateField
 from langflow.utils import util
+from langflow.utils.util import build_template_from_class
 
 TOOL_INPUTS = {
     "str": TemplateField(
@@ -66,64 +64,81 @@ class ToolCreator(LangChainTypeCreator):
     @property
     def type_to_loader_dict(self) -> Dict:
         if self.tools_dict is None:
-            self.tools_dict = get_tools_dict()
+            all_tools = {}
+            for tool, tool_fcn in ALL_TOOLS_NAMES.items():
+                tool_params = get_tool_params(tool_fcn)
+                tool_name = tool_params.get("name", tool)
+
+                if tool_name in settings.tools or settings.dev:
+                    if tool_name == "JsonSpec":
+                        tool_params["path"] = tool_params.pop("dict_")  # type: ignore
+                    all_tools[tool_name] = {
+                        "type": tool,
+                        "params": tool_params,
+                        "fcn": tool_fcn,
+                    }
+
+            self.tools_dict = all_tools
+
         return self.tools_dict
 
     def get_signature(self, name: str) -> Optional[Dict]:
         """Get the signature of a tool."""
 
         base_classes = ["Tool"]
-        all_tools = {}
-        for tool in self.type_to_loader_dict.keys():
-            tool_fcn = get_tool_by_name(tool)
-            if tool_params := get_tool_params(tool_fcn):
-                tool_name = tool_params.get("name") or str(tool)
-                all_tools[tool_name] = {
-                    "type": tool,
-                    "params": tool_params,
-                    "fcn": tool_fcn,
-                }
+        fields = []
+        params = []
+        tool_params = {}
 
         # Raise error if name is not in tools
-        if name not in all_tools.keys():
+        if name not in self.type_to_loader_dict.keys():
             raise ValueError("Tool not found")
 
-        tool_type: str = all_tools[name]["type"]  # type: ignore
+        tool_type: str = self.type_to_loader_dict[name]["type"]  # type: ignore
 
-        if all_tools[tool_type]["fcn"] in _BASE_TOOLS.values():
-            params = []
-        elif all_tools[tool_type]["fcn"] in _LLM_TOOLS.values():
+        # if tool_type in _BASE_TOOLS.keys():
+        #     params = []
+        if tool_type in _LLM_TOOLS.keys():
             params = ["llm"]
-        elif all_tools[tool_type]["fcn"] in [
-            val[0] for val in _EXTRA_LLM_TOOLS.values()
-        ]:
-            n_dict = {val[0]: val[1] for val in _EXTRA_LLM_TOOLS.values()}
-            extra_keys = n_dict[all_tools[tool_type]["fcn"]]
+        elif tool_type in _EXTRA_LLM_TOOLS.keys():
+            extra_keys = _EXTRA_LLM_TOOLS[tool_type][1]
             params = ["llm"] + extra_keys
-        elif all_tools[tool_type]["fcn"] in [
-            val[0] for val in _EXTRA_OPTIONAL_TOOLS.values()
-        ]:
-            n_dict = {val[0]: val[1] for val in _EXTRA_OPTIONAL_TOOLS.values()}  # type: ignore
-            extra_keys = n_dict[all_tools[tool_type]["fcn"]]
+        elif tool_type in _EXTRA_OPTIONAL_TOOLS.keys():
+            extra_keys = _EXTRA_OPTIONAL_TOOLS[tool_type][1]
             params = extra_keys
         # elif tool_type == "Tool":
         #     params = ["name", "description", "func"]
         elif tool_type in CUSTOM_TOOLS:
             # Get custom tool params
-            params = all_tools[name]["params"]  # type: ignore
+            params = self.type_to_loader_dict[name]["params"]  # type: ignore
             base_classes = ["function"]
             if node := customs.get_custom_nodes("tools").get(tool_type):
                 return node
         elif tool_type in FILE_TOOLS:
-            params = all_tools[name]["params"]  # type: ignore
-            if tool_type == "JsonSpec":
-                params["path"] = params.pop("dict_")  # type: ignore
+            params = self.type_to_loader_dict[name]["params"]  # type: ignore
             base_classes += [name]
-        else:
-            params = []
+        elif tool_type in OTHER_TOOLS:
+            print(tool_type)
+            tool_dict = build_template_from_class(tool_type, OTHER_TOOLS)
+            fields = tool_dict["template"]
+
+            # Pop unnecessary fields and add name
+            fields.pop("_type")  # type: ignore
+            fields.pop("return_direct")  # type: ignore
+            fields.pop("verbose")  # type: ignore
+
+            tool_params = {
+                "name": fields.pop("name")["value"],  # type: ignore
+                "description": fields.pop("description")["value"],  # type: ignore
+            }
+
+            fields = [
+                TemplateField(name=name, field_type=field["type"], **field)
+                for name, field in fields.items()  # type: ignore
+            ]
+            base_classes += tool_dict["base_classes"]
 
         # Copy the field and add the name
-        fields = []
         for param in params:
             field = TOOL_INPUTS.get(param, TOOL_INPUTS["str"]).copy()
             field.name = param
@@ -134,7 +149,7 @@ class ToolCreator(LangChainTypeCreator):
 
         template = Template(fields=fields, type_name=tool_type)
 
-        tool_params = all_tools[name]["params"]
+        tool_params = {**tool_params, **self.type_to_loader_dict[name]["params"]}
         return {
             "template": util.format_dict(template.to_dict()),
             **tool_params,
@@ -144,21 +159,7 @@ class ToolCreator(LangChainTypeCreator):
     def to_list(self) -> List[str]:
         """List all load tools"""
 
-        tools = []
-
-        for tool, fcn in get_tools_dict().items():
-            tool_params = get_tool_params(fcn)
-
-            if tool_params and not tool_params.get("name"):
-                tool_params["name"] = tool
-
-            if tool_params and (
-                tool_params.get("name") in settings.tools
-                or (tool_params.get("name") and settings.dev)
-            ):
-                tools.append(tool_params["name"])
-
-        return tools
+        return list(self.type_to_loader_dict.keys())
 
 
 tool_creator = ToolCreator()

src/backend/langflow/interface/tools/constants.py

@@ -5,12 +5,50 @@ from langchain.agents.load_tools import (
     _EXTRA_OPTIONAL_TOOLS,
     _LLM_TOOLS,
 )
-from langchain.tools.json.tool import JsonSpec
+from langchain.tools.bing_search.tool import BingSearchRun
+from langchain.tools.google_search.tool import GoogleSearchResults, GoogleSearchRun
+from langchain.tools.json.tool import JsonGetValueTool, JsonListKeysTool, JsonSpec
+from langchain.tools.python.tool import PythonAstREPLTool, PythonREPLTool
+from langchain.tools.requests.tool import (
+    RequestsDeleteTool,
+    RequestsGetTool,
+    RequestsPatchTool,
+    RequestsPostTool,
+    RequestsPutTool,
+)
+from langchain.tools.sql_database.tool import (
+    InfoSQLDatabaseTool,
+    ListSQLDatabaseTool,
+    QueryCheckerTool,
+    QuerySQLDataBaseTool,
+)
+from langchain.tools.wikipedia.tool import WikipediaQueryRun
+from langchain.tools.wolfram_alpha.tool import WolframAlphaQueryRun
 
 from langflow.interface.tools.custom import PythonFunction
 
 FILE_TOOLS = {"JsonSpec": JsonSpec}
 CUSTOM_TOOLS = {"Tool": Tool, "PythonFunction": PythonFunction}
+OTHER_TOOLS = {
+    "QuerySQLDataBaseTool": QuerySQLDataBaseTool,
+    "InfoSQLDatabaseTool": InfoSQLDatabaseTool,
+    "ListSQLDatabaseTool": ListSQLDatabaseTool,
+    "QueryCheckerTool": QueryCheckerTool,
+    "BingSearchRun": BingSearchRun,
+    "GoogleSearchRun": GoogleSearchRun,
+    "GoogleSearchResults": GoogleSearchResults,
+    "JsonListKeysTool": JsonListKeysTool,
+    "JsonGetValueTool": JsonGetValueTool,
+    "PythonREPLTool": PythonREPLTool,
+    "PythonAstREPLTool": PythonAstREPLTool,
+    "RequestsGetTool": RequestsGetTool,
+    "RequestsPostTool": RequestsPostTool,
+    "RequestsPatchTool": RequestsPatchTool,
+    "RequestsPutTool": RequestsPutTool,
+    "RequestsDeleteTool": RequestsDeleteTool,
+    "WikipediaQueryRun": WikipediaQueryRun,
+    "WolframAlphaQueryRun": WolframAlphaQueryRun,
+}
 ALL_TOOLS_NAMES = {
     **_BASE_TOOLS,
     **_LLM_TOOLS,  # type: ignore
@@ -18,4 +56,5 @@ ALL_TOOLS_NAMES = {
     **{k: v[0] for k, v in _EXTRA_OPTIONAL_TOOLS.items()},
     **CUSTOM_TOOLS,
     **FILE_TOOLS,  # type: ignore
+    **OTHER_TOOLS,
 }

src/backend/langflow/interface/tools/util.py

@@ -4,29 +4,6 @@ from typing import Dict, Union
 
 from langchain.agents.tools import Tool
 
-from langflow.interface.tools.constants import ALL_TOOLS_NAMES
-
-
-def get_tools_dict():
-    """Get the tools dictionary."""
-
-    all_tools = {}
-
-    for tool, fcn in ALL_TOOLS_NAMES.items():
-        if tool_params := get_tool_params(fcn):
-            tool_name = tool_params.get("name") or str(tool)
-            all_tools[tool_name] = fcn
-
-    return all_tools
-
-
-def get_tool_by_name(name: str):
-    """Get a tool from the tools dictionary."""
-    tools = get_tools_dict()
-    if name not in tools:
-        raise ValueError(f"{name} not found.")
-    return tools[name]
-
 
 def get_func_tool_params(func, **kwargs) -> Union[Dict, None]:
     tree = ast.parse(inspect.getsource(func))
@@ -113,6 +90,8 @@ def get_tool_params(tool, **kwargs) -> Dict:
     elif inspect.isclass(tool):
         # Get the parameters necessary to
         # instantiate the class
+
         return get_class_tool_params(tool, **kwargs) or {}
+
     else:
         raise ValueError("Tool must be a function or class.")

src/backend/langflow/interface/types.py

@@ -8,6 +8,7 @@ from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
+from langflow.interface.utilities.base import utility_creator
 from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
@@ -42,6 +43,7 @@ def build_langchain_types_dict():  # sourcery skip: dict-assign-update-to-union
         vectorstore_creator,
         documentloader_creator,
         textsplitter_creator,
+        utility_creator,
     ]
 
     all_types = {}

src/backend/langflow/interface/utilities/base.py

@@ -0,0 +1,39 @@
+from typing import Dict, List, Optional
+
+from langflow.custom.customs import get_custom_nodes
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import utility_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
+
+
+class UtilityCreator(LangChainTypeCreator):
+    type_name: str = "utilities"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return utility_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Optional[Dict]:
+        """Get the signature of a utility."""
+        try:
+            if name in get_custom_nodes(self.type_name).keys():
+                return get_custom_nodes(self.type_name)[name]
+            return build_template_from_class(name, utility_type_to_cls_dict)
+        except ValueError as exc:
+            raise ValueError(f"Utility {name} not found") from exc
+
+        except AttributeError as exc:
+            logger.error(f"Utility {name} not loaded: {exc}")
+            return None
+
+    def to_list(self) -> List[str]:
+        return [
+            utility.__name__
+            for utility in self.type_to_loader_dict.values()
+            if utility.__name__ in settings.utilities or settings.dev
+        ]
+
+
+utility_creator = UtilityCreator()

src/backend/langflow/settings.py

@@ -18,6 +18,7 @@ class Settings(BaseSettings):
     wrappers: List[str] = []
     toolkits: List[str] = []
     textsplitters: List[str] = []
+    utilities: List[str] = []
     dev: bool = False
 
     class Config:
@@ -42,6 +43,7 @@ class Settings(BaseSettings):
         self.wrappers = new_settings.wrappers or []
         self.toolkits = new_settings.toolkits or []
         self.textsplitters = new_settings.textsplitters or []
+        self.utilities = new_settings.utilities or []
         self.dev = new_settings.dev or False
 
 

src/backend/langflow/template/nodes.py

@@ -256,6 +256,29 @@ class CSVAgentNode(FrontendNode):
         return super().to_dict()
 
 
+class SQLDatabaseNode(FrontendNode):
+    name: str = "SQLDatabase"
+    template: Template = Template(
+        type_name="sql_database",
+        fields=[
+            TemplateField(
+                field_type="str",
+                required=True,
+                is_list=False,
+                show=True,
+                multiline=False,
+                value="",
+                name="uri",
+            ),
+        ],
+    )
+    description: str = """SQLAlchemy wrapper around a database."""
+    base_classes: list[str] = ["SQLDatabase"]
+
+    def to_dict(self):
+        return super().to_dict()
+
+
 class VectorStoreAgentNode(FrontendNode):
     name: str = "VectorStoreAgent"
     template: Template = Template(

src/frontend/src/utils.ts

@@ -13,7 +13,8 @@ import {
   QuestionMarkCircleIcon,
   FingerPrintIcon,
   ScissorsIcon,
-  CircleStackIcon
+  CircleStackIcon,
+  Squares2X2Icon
 } from "@heroicons/react/24/outline";
 import { Connection, Edge, Node, ReactFlowInstance } from "reactflow";
 import { FlowType } from "./types/flow";
@@ -86,6 +87,7 @@ export const nodeColors: {[char: string]: string} = {
   textsplitters: "#B47CB5",
   toolkits:"#DB2C2C",
   wrappers:"#E6277A",
+  utilities:"#31A3CC",
   unknown:"#9CA3AF"
 };
 
@@ -104,6 +106,7 @@ export const nodeNames:{[char: string]: string} = {
   toolkits:"Toolkits",
   wrappers:"Wrappers",
   textsplitters: "Text Splitters",
+  utilities:"Utilities",
   unknown:"Unknown"
 };
 
@@ -122,6 +125,7 @@ export const nodeIcons:{[char: string]: React.ForwardRefExoticComponent<React.SV
   toolkits:WrenchScrewdriverIcon,
   textsplitters:ScissorsIcon,
   wrappers:GiftIcon,
+  utilities:Squares2X2Icon,
   unknown:QuestionMarkCircleIcon
 };