From 843ae8efc50253bdea3fd9385a5e646a905ab34e Mon Sep 17 00:00:00 2001 From: Ibis Prevedello Date: Mon, 17 Apr 2023 23:15:39 -0300 Subject: [PATCH] feat: add extra tools and utilities --- src/backend/langflow/config.yaml | 28 +++++++++++++ .../langflow/interface/custom_lists.py | 6 +++ src/backend/langflow/interface/listing.py | 2 + src/backend/langflow/interface/tools/base.py | 14 ++++--- .../langflow/interface/tools/constants.py | 42 +++++++++++++++++-- src/backend/langflow/interface/tools/util.py | 1 - src/backend/langflow/interface/types.py | 2 + .../langflow/interface/utilities/__init__.py | 0 .../langflow/interface/utilities/base.py | 36 ++++++++++++++++ src/backend/langflow/settings.py | 2 + src/frontend/src/utils.ts | 6 ++- 11 files changed, 128 insertions(+), 11 deletions(-) create mode 100644 src/backend/langflow/interface/utilities/__init__.py create mode 100644 src/backend/langflow/interface/utilities/base.py diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 3508f5196..605df68ad 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -44,6 +44,23 @@ tools: - TMDB API - Podcast API - QuerySQLDataBaseTool + - InfoSQLDatabaseTool + - ListSQLDatabaseTool + # - QueryCheckerTool + - BingSearchRun + - GoogleSearchRun + - GoogleSearchResults + - JsonListKeysTool + - JsonGetValueTool + - PythonREPLTool + - PythonAstREPLTool + - RequestsGetTool + - RequestsPostTool + - RequestsPatchTool + - RequestsPutTool + - RequestsDeleteTool + - WikipediaQueryRun + - WolframAlphaQueryRun wrappers: - RequestsWrapper @@ -95,4 +112,15 @@ documentloaders: textsplitters: - CharacterTextSplitter +utilities: + - BingSearchAPIWrapper + - GoogleSearchAPIWrapper + - GoogleSerperAPIWrapper + - SearxResults + - SearxSearchWrapper + - SerpAPIWrapper + - WikipediaAPIWrapper + - WolframAlphaAPIWrapper + # - ZapierNLAWrapper + dev: false diff --git a/src/backend/langflow/interface/custom_lists.py b/src/backend/langflow/interface/custom_lists.py index 2ffc18b14..fb97e8dae 100644 --- a/src/backend/langflow/interface/custom_lists.py +++ b/src/backend/langflow/interface/custom_lists.py @@ -9,6 +9,7 @@ from langchain import ( memory, requests, text_splitter, + utilities, vectorstores, ) from langchain.agents import agent_toolkits @@ -76,3 +77,8 @@ documentloaders_type_to_cls_dict: dict[str, Any] = { textsplitter_type_to_cls_dict: dict[str, Any] = dict( inspect.getmembers(text_splitter, inspect.isclass) ) + +## Utilities +utility_type_to_cls_dict: dict[str, Any] = dict( + inspect.getmembers(utilities, inspect.isclass) +) diff --git a/src/backend/langflow/interface/listing.py b/src/backend/langflow/interface/listing.py index cf45fd9c5..3d73105c2 100644 --- a/src/backend/langflow/interface/listing.py +++ b/src/backend/langflow/interface/listing.py @@ -8,6 +8,7 @@ from langflow.interface.prompts.base import prompt_creator from langflow.interface.text_splitters.base import textsplitter_creator from langflow.interface.toolkits.base import toolkits_creator from langflow.interface.tools.base import tool_creator +from langflow.interface.utilities.base import utility_creator from langflow.interface.vector_store.base import vectorstore_creator from langflow.interface.wrappers.base import wrapper_creator @@ -26,6 +27,7 @@ def get_type_dict(): "vectorStore": vectorstore_creator.to_list(), "embeddings": embedding_creator.to_list(), "textSplitters": textsplitter_creator.to_list(), + "utilities": utility_creator.to_list(), } diff --git a/src/backend/langflow/interface/tools/base.py b/src/backend/langflow/interface/tools/base.py index a5745f1d3..f236831d8 100644 --- a/src/backend/langflow/interface/tools/base.py +++ b/src/backend/langflow/interface/tools/base.py @@ -119,21 +119,23 @@ class ToolCreator(LangChainTypeCreator): params = self.type_to_loader_dict[name]["params"] # type: ignore base_classes += [name] elif tool_type in OTHER_TOOLS: + print(tool_type) tool_dict = build_template_from_class(tool_type, OTHER_TOOLS) fields = tool_dict["template"] # Pop unnecessary fields and add name - fields.pop("_type") - fields.pop("return_direct") - fields.pop("verbose") + fields.pop("_type") # type: ignore + fields.pop("return_direct") # type: ignore + fields.pop("verbose") # type: ignore tool_params = { - "name": fields.pop("name")["value"], - "description": fields.pop("description")["value"], + "name": fields.pop("name")["value"], # type: ignore + "description": fields.pop("description")["value"], # type: ignore } fields = [ - TemplateField(name=name, **field) for name, field in fields.items() + TemplateField(name=name, field_type=field["type"], **field) + for name, field in fields.items() # type: ignore ] base_classes += tool_dict["base_classes"] diff --git a/src/backend/langflow/interface/tools/constants.py b/src/backend/langflow/interface/tools/constants.py index 101fefa8b..34890a684 100644 --- a/src/backend/langflow/interface/tools/constants.py +++ b/src/backend/langflow/interface/tools/constants.py @@ -5,14 +5,50 @@ from langchain.agents.load_tools import ( _EXTRA_OPTIONAL_TOOLS, _LLM_TOOLS, ) -from langchain.tools.json.tool import JsonSpec -from langchain.tools.sql_database.tool import QuerySQLDataBaseTool +from langchain.tools.bing_search.tool import BingSearchRun +from langchain.tools.google_search.tool import GoogleSearchResults, GoogleSearchRun +from langchain.tools.json.tool import JsonGetValueTool, JsonListKeysTool, JsonSpec +from langchain.tools.python.tool import PythonAstREPLTool, PythonREPLTool +from langchain.tools.requests.tool import ( + RequestsDeleteTool, + RequestsGetTool, + RequestsPatchTool, + RequestsPostTool, + RequestsPutTool, +) +from langchain.tools.sql_database.tool import ( + InfoSQLDatabaseTool, + ListSQLDatabaseTool, + QueryCheckerTool, + QuerySQLDataBaseTool, +) +from langchain.tools.wikipedia.tool import WikipediaQueryRun +from langchain.tools.wolfram_alpha.tool import WolframAlphaQueryRun from langflow.interface.tools.custom import PythonFunction FILE_TOOLS = {"JsonSpec": JsonSpec} CUSTOM_TOOLS = {"Tool": Tool, "PythonFunction": PythonFunction} -OTHER_TOOLS = {"QuerySQLDataBaseTool": QuerySQLDataBaseTool} +OTHER_TOOLS = { + "QuerySQLDataBaseTool": QuerySQLDataBaseTool, + "InfoSQLDatabaseTool": InfoSQLDatabaseTool, + "ListSQLDatabaseTool": ListSQLDatabaseTool, + "QueryCheckerTool": QueryCheckerTool, + "BingSearchRun": BingSearchRun, + "GoogleSearchRun": GoogleSearchRun, + "GoogleSearchResults": GoogleSearchResults, + "JsonListKeysTool": JsonListKeysTool, + "JsonGetValueTool": JsonGetValueTool, + "PythonREPLTool": PythonREPLTool, + "PythonAstREPLTool": PythonAstREPLTool, + "RequestsGetTool": RequestsGetTool, + "RequestsPostTool": RequestsPostTool, + "RequestsPatchTool": RequestsPatchTool, + "RequestsPutTool": RequestsPutTool, + "RequestsDeleteTool": RequestsDeleteTool, + "WikipediaQueryRun": WikipediaQueryRun, + "WolframAlphaQueryRun": WolframAlphaQueryRun, +} ALL_TOOLS_NAMES = { **_BASE_TOOLS, **_LLM_TOOLS, # type: ignore diff --git a/src/backend/langflow/interface/tools/util.py b/src/backend/langflow/interface/tools/util.py index 41fb4bdb7..f1d66696a 100644 --- a/src/backend/langflow/interface/tools/util.py +++ b/src/backend/langflow/interface/tools/util.py @@ -5,7 +5,6 @@ from typing import Dict, Union from langchain.agents.tools import Tool - def get_func_tool_params(func, **kwargs) -> Union[Dict, None]: tree = ast.parse(inspect.getsource(func)) diff --git a/src/backend/langflow/interface/types.py b/src/backend/langflow/interface/types.py index fd5d9ec3e..085537756 100644 --- a/src/backend/langflow/interface/types.py +++ b/src/backend/langflow/interface/types.py @@ -8,6 +8,7 @@ from langflow.interface.prompts.base import prompt_creator from langflow.interface.text_splitters.base import textsplitter_creator from langflow.interface.toolkits.base import toolkits_creator from langflow.interface.tools.base import tool_creator +from langflow.interface.utilities.base import utility_creator from langflow.interface.vector_store.base import vectorstore_creator from langflow.interface.wrappers.base import wrapper_creator @@ -42,6 +43,7 @@ def build_langchain_types_dict(): # sourcery skip: dict-assign-update-to-union vectorstore_creator, documentloader_creator, textsplitter_creator, + utility_creator, ] all_types = {} diff --git a/src/backend/langflow/interface/utilities/__init__.py b/src/backend/langflow/interface/utilities/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/langflow/interface/utilities/base.py b/src/backend/langflow/interface/utilities/base.py new file mode 100644 index 000000000..a56e8bce8 --- /dev/null +++ b/src/backend/langflow/interface/utilities/base.py @@ -0,0 +1,36 @@ +from typing import Dict, List, Optional + +from langflow.interface.base import LangChainTypeCreator +from langflow.interface.custom_lists import utility_type_to_cls_dict +from langflow.settings import settings +from langflow.utils.logger import logger +from langflow.utils.util import build_template_from_class + + +class UtilityCreator(LangChainTypeCreator): + type_name: str = "utilities" + + @property + def type_to_loader_dict(self) -> Dict: + return utility_type_to_cls_dict + + def get_signature(self, name: str) -> Optional[Dict]: + """Get the signature of a utility.""" + try: + return build_template_from_class(name, utility_type_to_cls_dict) + except ValueError as exc: + raise ValueError(f"Utility {name} not found") from exc + + except AttributeError as exc: + logger.error(f"Utility {name} not loaded: {exc}") + return None + + def to_list(self) -> List[str]: + return [ + utility.__name__ + for utility in self.type_to_loader_dict.values() + if utility.__name__ in settings.utilities or settings.dev + ] + + +utility_creator = UtilityCreator() diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py index c5377c85a..48aa5939d 100644 --- a/src/backend/langflow/settings.py +++ b/src/backend/langflow/settings.py @@ -18,6 +18,7 @@ class Settings(BaseSettings): wrappers: List[str] = [] toolkits: List[str] = [] textsplitters: List[str] = [] + utilities: List[str] = [] dev: bool = False class Config: @@ -42,6 +43,7 @@ class Settings(BaseSettings): self.wrappers = new_settings.wrappers or [] self.toolkits = new_settings.toolkits or [] self.textsplitters = new_settings.textsplitters or [] + self.utilities = new_settings.utilities or [] self.dev = new_settings.dev or False diff --git a/src/frontend/src/utils.ts b/src/frontend/src/utils.ts index 5ab47f31e..405d56297 100644 --- a/src/frontend/src/utils.ts +++ b/src/frontend/src/utils.ts @@ -13,7 +13,8 @@ import { QuestionMarkCircleIcon, FingerPrintIcon, ScissorsIcon, - CircleStackIcon + CircleStackIcon, + Squares2X2Icon } from "@heroicons/react/24/outline"; import { Connection, Edge, Node, ReactFlowInstance } from "reactflow"; import { FlowType } from "./types/flow"; @@ -85,6 +86,7 @@ export const nodeColors: {[char: string]: string} = { textsplitters: "#B47CB5", toolkits:"#DB2C2C", wrappers:"#E6277A", + utilities:"#31A3CC", unknown:"#9CA3AF" }; @@ -103,6 +105,7 @@ export const nodeNames:{[char: string]: string} = { toolkits:"Toolkits", wrappers:"Wrappers", textsplitters: "Text Splitters", + utilities:"Utilities", unknown:"Unknown" }; @@ -121,6 +124,7 @@ export const nodeIcons:{[char: string]: React.ForwardRefExoticComponent