From a9c7fc0a6983c942b96b45f0158cbaecad11f74f Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 00:38:14 -0300 Subject: [PATCH 001/442] =?UTF-8?q?=F0=9F=94=A7=20chore(config):=20add=20V?= =?UTF-8?q?ectorStoreToolkit=20to=20toolkits=20list=20=F0=9F=90=9B=20fix(b?= =?UTF-8?q?ase.py):=20remove=20deepcopy=20for=20VectorStore=20and=20Vector?= =?UTF-8?q?StoreRouter=20agents=20=F0=9F=90=9B=20fix(nodes.py):=20remove?= =?UTF-8?q?=20deepcopy=20for=20VectorStore=20and=20VectorStoreRouter=20age?= =?UTF-8?q?nts=20=F0=9F=94=A7=20chore(loading.py):=20comment=20out=20unuse?= =?UTF-8?q?d=20code=20for=20loading=20toolkits=20=F0=9F=90=9B=20fix(toolki?= =?UTF-8?q?ts/base.py):=20add=20Tool=20to=20base=5Fclasses=20in=20get=5Fsi?= =?UTF-8?q?gnature=20method=20The=20changes=20to=20the=20config=20file=20a?= =?UTF-8?q?dd=20the=20VectorStoreToolkit=20to=20the=20list=20of=20toolkits?= =?UTF-8?q?.=20The=20deepcopy=20for=20VectorStore=20and=20VectorStoreRoute?= =?UTF-8?q?r=20agents=20was=20causing=20issues,=20so=20it=20was=20removed?= =?UTF-8?q?=20from=20the=20base.py=20and=20nodes.py=20files.=20The=20loadi?= =?UTF-8?q?ng.py=20file=20had=20some=20unused=20code=20for=20loading=20too?= =?UTF-8?q?lkits,=20so=20it=20was=20commented=20out.=20Finally,=20the=20ba?= =?UTF-8?q?se.py=20file=20had=20a=20bug=20where=20the=20Tool=20class=20was?= =?UTF-8?q?=20not=20being=20added=20to=20the=20base=5Fclasses=20list=20in?= =?UTF-8?q?=20the=20get=5Fsignature=20method,=20so=20it=20was=20added.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/config.yaml | 1 + src/backend/langflow/graph/base.py | 14 +------------- src/backend/langflow/graph/nodes.py | 5 +---- src/backend/langflow/interface/loading.py | 7 +++++-- src/backend/langflow/interface/toolkits/base.py | 15 +++++++++------ 5 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 02b17cd85..b073ed544 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -74,6 +74,7 @@ toolkits: - JsonToolkit - VectorStoreInfo - VectorStoreRouterToolkit + - VectorStoreToolkit tools: - Search - PAL-MATH diff --git a/src/backend/langflow/graph/base.py b/src/backend/langflow/graph/base.py index 187d2983e..5b64885fb 100644 --- a/src/backend/langflow/graph/base.py +++ b/src/backend/langflow/graph/base.py @@ -212,19 +212,7 @@ class Node: if not self._built or force: self._build() - #! Deepcopy is breaking for vectorstores - if self.base_type in [ - "vectorstores", - "VectorStoreRouterAgent", - "VectorStoreAgent", - "VectorStoreInfo", - ] or self.node_type in [ - "VectorStoreInfo", - "VectorStoreRouterToolkit", - "SQLDatabase", - ]: - return self._built_object - return deepcopy(self._built_object) + return self._built_object def add_edge(self, edge: "Edge") -> None: self.edges.append(edge) diff --git a/src/backend/langflow/graph/nodes.py b/src/backend/langflow/graph/nodes.py index 189e40b5c..7d9b05366 100644 --- a/src/backend/langflow/graph/nodes.py +++ b/src/backend/langflow/graph/nodes.py @@ -14,7 +14,7 @@ class AgentNode(Node): def _set_tools_and_chains(self) -> None: for edge in self.edges: source_node = edge.source - if isinstance(source_node, ToolNode): + if isinstance(source_node, (ToolNode, ToolkitNode)): self.tools.append(source_node) elif isinstance(source_node, ChainNode): self.chains.append(source_node) @@ -32,9 +32,6 @@ class AgentNode(Node): self._build() - #! Cannot deepcopy VectorStore, VectorStoreRouter, or SQL agents - if self.node_type in ["VectorStoreAgent", "VectorStoreRouterAgent", "SQLAgent"]: - return self._built_object return self._built_object diff --git a/src/backend/langflow/interface/loading.py b/src/backend/langflow/interface/loading.py index cd6898a7f..d720c6b0c 100644 --- a/src/backend/langflow/interface/loading.py +++ b/src/backend/langflow/interface/loading.py @@ -101,8 +101,11 @@ def instantiate_tool(node_type, class_object, params): def instantiate_toolkit(node_type, class_object, params): loaded_toolkit = class_object(**params) - if toolkits_creator.has_create_function(node_type): - return load_toolkits_executor(node_type, loaded_toolkit, params) + # Commenting this out for now to use toolkits as normal tools + # if toolkits_creator.has_create_function(node_type): + # return load_toolkits_executor(node_type, loaded_toolkit, params) + if isinstance(loaded_toolkit, BaseToolkit): + return loaded_toolkit.get_tools() return loaded_toolkit diff --git a/src/backend/langflow/interface/toolkits/base.py b/src/backend/langflow/interface/toolkits/base.py index cbe625f0d..9f01b2bb2 100644 --- a/src/backend/langflow/interface/toolkits/base.py +++ b/src/backend/langflow/interface/toolkits/base.py @@ -42,24 +42,27 @@ class ToolkitCreator(LangChainTypeCreator): def get_signature(self, name: str) -> Optional[Dict]: try: - return build_template_from_class(name, self.type_to_loader_dict) + template = build_template_from_class(name, self.type_to_loader_dict) + # add Tool to base_classes + if template: + template["base_classes"].append("Tool") + return template except ValueError as exc: - raise ValueError("Prompt not found") from exc + raise ValueError("Toolkit not found") from exc except AttributeError as exc: - logger.error(f"Prompt {name} not loaded: {exc}") + logger.error(f"Toolkit {name} not loaded: {exc}") return None def to_list(self) -> List[str]: return list(self.type_to_loader_dict.keys()) def get_create_function(self, name: str) -> Callable: - if loader_name := self.create_functions.get(name, None): - # import loader + if loader_name := self.create_functions.get(name): return import_module( f"from langchain.agents.agent_toolkits import {loader_name[0]}" ) else: - raise ValueError("Loader not found") + raise ValueError("Toolkit not found") def has_create_function(self, name: str) -> bool: # check if the function list is not empty From ad3bb997eed981c9426e0b76b16f84309b18ae74 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 01:11:27 -0300 Subject: [PATCH 002/442] =?UTF-8?q?=F0=9F=90=9B=20fix(base.py):=20extend?= =?UTF-8?q?=20list=20only=20if=20key=20exists=20and=20is=20a=20list=20?= =?UTF-8?q?=F0=9F=90=9B=20fix(nodes.py):=20flatten=20list=20of=20tools=20i?= =?UTF-8?q?f=20it=20is=20a=20list=20of=20lists=20=F0=9F=90=9B=20fix(toolki?= =?UTF-8?q?ts/base.py):=20add=20"toolkit"=20check=20to=20avoid=20adding=20?= =?UTF-8?q?"Tool"=20to=20non-toolkit=20classes=20=F0=9F=93=9D=20docs(agent?= =?UTF-8?q?s.py):=20update=20node=20descriptions=20to=20reflect=20CSV=20an?= =?UTF-8?q?d=20zero=20shot=20agents=20The=20changes=20in=20base.py=20and?= =?UTF-8?q?=20nodes.py=20ensure=20that=20the=20code=20works=20as=20intende?= =?UTF-8?q?d=20and=20avoids=20errors=20when=20extending=20lists.=20The=20c?= =?UTF-8?q?hange=20in=20toolkits/base.py=20ensures=20that=20"Tool"=20is=20?= =?UTF-8?q?only=20added=20to=20classes=20that=20are=20toolkits.=20The=20ch?= =?UTF-8?q?anges=20in=20agents.py=20update=20the=20node=20descriptions=20t?= =?UTF-8?q?o=20reflect=20that=20the=20CSVAgentNode=20constructs=20a=20CSV?= =?UTF-8?q?=20agent=20and=20the=20InitializeAgentNode=20constructs=20a=20z?= =?UTF-8?q?ero=20shot=20agent.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/graph/base.py | 6 ++++++ src/backend/langflow/graph/nodes.py | 5 +++++ src/backend/langflow/interface/toolkits/base.py | 2 +- src/backend/langflow/template/frontend_node/agents.py | 4 ++-- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/backend/langflow/graph/base.py b/src/backend/langflow/graph/base.py index 5b64885fb..08b255441 100644 --- a/src/backend/langflow/graph/base.py +++ b/src/backend/langflow/graph/base.py @@ -175,6 +175,12 @@ class Node: # turn result which is a function into a coroutine # so that it can be awaited self.params["coroutine"] = sync_to_async(result) + if isinstance(result, list): + # If the result is a list, then we need to extend the list + # with the result but first check if the key exists + # if it doesn't, then we need to create a new list + if isinstance(self.params[key], list): + self.params[key].extend(result) self.params[key] = result elif isinstance(value, list) and all( diff --git a/src/backend/langflow/graph/nodes.py b/src/backend/langflow/graph/nodes.py index 7d9b05366..ea94e10b8 100644 --- a/src/backend/langflow/graph/nodes.py +++ b/src/backend/langflow/graph/nodes.py @@ -62,6 +62,11 @@ class PromptNode(Node): if tools is not None else [] ) + # flatten the list of tools if it is a list of lists + # first check if it is a list + if isinstance(tools, list) and isinstance(tools[0], list): + tools = [tool for sublist in tools for tool in sublist] + self.params["tools"] = tools prompt_params = [ key diff --git a/src/backend/langflow/interface/toolkits/base.py b/src/backend/langflow/interface/toolkits/base.py index 9f01b2bb2..be2345c02 100644 --- a/src/backend/langflow/interface/toolkits/base.py +++ b/src/backend/langflow/interface/toolkits/base.py @@ -44,7 +44,7 @@ class ToolkitCreator(LangChainTypeCreator): try: template = build_template_from_class(name, self.type_to_loader_dict) # add Tool to base_classes - if template: + if "toolkit" in name.lower() and template: template["base_classes"].append("Tool") return template except ValueError as exc: diff --git a/src/backend/langflow/template/frontend_node/agents.py b/src/backend/langflow/template/frontend_node/agents.py index e4fe40187..451dd7eca 100644 --- a/src/backend/langflow/template/frontend_node/agents.py +++ b/src/backend/langflow/template/frontend_node/agents.py @@ -146,7 +146,7 @@ class CSVAgentNode(FrontendNode): ), ], ) - description: str = """Construct a json agent from a CSV and tools.""" + description: str = """Construct a CSV agent from a CSV and tools.""" base_classes: list[str] = ["AgentExecutor"] def to_dict(self): @@ -194,7 +194,7 @@ class InitializeAgentNode(FrontendNode): ), ], ) - description: str = """Construct a json agent from an LLM and tools.""" + description: str = """Construct a zero shot agent from an LLM and tools.""" base_classes: list[str] = ["AgentExecutor", "function"] def to_dict(self): From 1a8d5561e9dbbecd636495ca3a75aecdaa2f3623 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 01:15:48 -0300 Subject: [PATCH 003/442] fix --- src/backend/langflow/graph/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/graph/nodes.py b/src/backend/langflow/graph/nodes.py index ea94e10b8..5f136b3c7 100644 --- a/src/backend/langflow/graph/nodes.py +++ b/src/backend/langflow/graph/nodes.py @@ -64,7 +64,7 @@ class PromptNode(Node): ) # flatten the list of tools if it is a list of lists # first check if it is a list - if isinstance(tools, list) and isinstance(tools[0], list): + if tools and isinstance(tools, list) and isinstance(tools[0], list): tools = [tool for sublist in tools for tool in sublist] self.params["tools"] = tools From 7edce61182473a647b4b549a1d389b2d379531b5 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Tue, 30 May 2023 17:58:13 -0300 Subject: [PATCH 004/442] Implemented add_extra_fields method for the documentloaders --- .../interface/document_loaders/base.py | 132 +++--------------- .../template/frontend_node/__init__.py | 4 + .../template/frontend_node/documentloaders.py | 122 ++++++++++++++++ .../template/frontend_node/textsplitters.py | 9 ++ .../template/frontend_node/vectorstores.py | 1 - 5 files changed, 156 insertions(+), 112 deletions(-) create mode 100644 src/backend/langflow/template/frontend_node/documentloaders.py create mode 100644 src/backend/langflow/template/frontend_node/textsplitters.py diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index a13d5cd5b..484db584c 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -1,134 +1,44 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode from langflow.interface.custom_lists import documentloaders_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class - def build_file_path_template( - suffixes: list, fileTypes: list, name: str = "file_path" -) -> Dict: - """Build a file path template for a document loader.""" - return { - "type": "file", - "required": True, - "show": True, - "name": name, - "value": "", - "suffixes": suffixes, - "fileTypes": fileTypes, - } - + suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a file path template for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" + @property + def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]: + return DocumentLoaderFrontNode + @property def type_to_loader_dict(self) -> Dict: return documentloaders_type_to_cls_dict - + def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a document loader.""" try: - signature = build_template_from_class( + return build_template_from_class( name, documentloaders_type_to_cls_dict ) - - file_path_templates = { - "AirbyteJSONLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": build_file_path_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": build_file_path_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredImageLoader": build_file_path_template( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], - ), - "UnstructuredMarkdownLoader": build_file_path_template( - suffixes=[".md"], fileTypes=["md"] - ), - "PyPDFLoader": build_file_path_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), - "UnstructuredPowerPointLoader": build_file_path_template( - suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] - ), - "SRTLoader": build_file_path_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "UnstructuredWordDocumentLoader": build_file_path_template( - suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] - ), - } - - if name in file_path_templates: - signature["template"]["file_path"] = file_path_templates[name] - elif name in { - "WebBaseLoader", - "AZLyricsLoader", - "CollegeConfidentialLoader", - "HNLoader", - "IFixitLoader", - "IMSDbLoader", - }: - signature["template"]["web_path"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_path", - "value": "", - "display_name": "Web Page", - } - elif name in {"GitbookLoader"}: - signature["template"]["web_page"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_page", - "value": "", - "display_name": "Web Page", - } - elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}: - signature["template"]["path"] = { - "type": "str", - "required": True, - "show": True, - "name": "path", - "value": "", - "display_name": "Web Page", - } - - return signature except ValueError as exc: raise ValueError(f"Documment Loader {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/template/frontend_node/__init__.py b/src/backend/langflow/template/frontend_node/__init__.py index 1aa946d41..c36234364 100644 --- a/src/backend/langflow/template/frontend_node/__init__.py +++ b/src/backend/langflow/template/frontend_node/__init__.py @@ -7,6 +7,8 @@ from langflow.template.frontend_node import ( prompts, tools, vectorstores, + documentloaders, + textsplitters, ) __all__ = [ @@ -18,4 +20,6 @@ __all__ = [ "llms", "prompts", "vectorstores", + "documentloaders", + "textsplitters", ] diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py new file mode 100644 index 000000000..d00072ad0 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -0,0 +1,122 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class DocumentLoaderFrontNode(FrontendNode): + + @staticmethod + def build_template(suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a template field for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) + + def get_file_path_template(self): + return { + "AirbyteJSONLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "CoNLLULoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "CSVLoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "UnstructuredEmailLoader": self.build_template( + suffixes=[".eml"], fileTypes=["eml"] + ), + "EverNoteLoader": self.build_template( + suffixes=[".xml"], fileTypes=["xml"] + ), + "FacebookChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "GutenbergLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "BSHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredImageLoader": self.build_template( + suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + ), + "UnstructuredMarkdownLoader": self.build_template( + suffixes=[".md"], fileTypes=["md"] + ), + "PyPDFLoader": self.build_template( + suffixes=[".pdf"], fileTypes=["pdf"] + ), + "UnstructuredPowerPointLoader": self.build_template( + suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] + ), + "SRTLoader": self.build_template( + suffixes=[".srt"], fileTypes=["srt"] + ), + "TelegramChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "TextLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "UnstructuredWordDocumentLoader": self.build_template( + suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] + ), + } + + def add_extra_fields(self) -> None: + file_path_templates = self.get_file_path_template() + + if self.template.type_name in file_path_templates: + self.template.add_field(file_path_templates[self.template.type_name]) + elif self.template.type_name in { + "WebBaseLoader", + "AZLyricsLoader", + "CollegeConfidentialLoader", + "HNLoader", + "IFixitLoader", + "IMSDbLoader", + }: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_path", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"GitbookLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_page", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"ReadTheDocsLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="path", + value="", + display_name="Web Page", + ) + ) \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py new file mode 100644 index 000000000..b5f2ce637 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -0,0 +1,9 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class TextSplittersFrontNode(FrontendNode): + + def add_extra_fields(self) -> None: + pass \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index 1aefaf10c..e071c3d4c 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -3,7 +3,6 @@ from typing import Optional from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode - class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: if self.template.type_name == "Weaviate": From 5291e981e2340e5bf36c7c713665411692d3a97f Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Tue, 30 May 2023 19:22:59 -0300 Subject: [PATCH 005/442] Implemented add_extra_fields for the documentloaders --- .../interface/document_loaders/base.py | 14 -- .../langflow/interface/text_splitters/base.py | 45 +------ .../template/frontend_node/documentloaders.py | 123 +++++++++--------- .../template/frontend_node/textsplitters.py | 41 +++++- 4 files changed, 107 insertions(+), 116 deletions(-) diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index 484db584c..0a0efbb71 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -8,20 +8,6 @@ from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class -def build_file_path_template( - suffixes: list, fileTypes: list, name: str = "file_path" - ) -> Dict: - """Build a file path template for a document loader.""" - return TemplateField( - field_type="file", - required=True, - show=True, - name=name, - value="", - suffixes=suffixes, - fileTypes=fileTypes, - ) - class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" diff --git a/src/backend/langflow/interface/text_splitters/base.py b/src/backend/langflow/interface/text_splitters/base.py index e58a5bfa6..11dfaca88 100644 --- a/src/backend/langflow/interface/text_splitters/base.py +++ b/src/backend/langflow/interface/text_splitters/base.py @@ -1,6 +1,7 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator +from langflow.template.frontend_node.textsplitters import VectorStoreFrontendNode from langflow.interface.custom_lists import textsplitter_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger @@ -10,6 +11,10 @@ from langflow.utils.util import build_template_from_class class TextSplitterCreator(LangChainTypeCreator): type_name: str = "textsplitters" + @property + def frontend_node_class(self) -> Type[VectorStoreFrontendNode]: + return VectorStoreFrontendNode + @property def type_to_loader_dict(self) -> Dict: return textsplitter_type_to_cls_dict @@ -17,43 +22,7 @@ class TextSplitterCreator(LangChainTypeCreator): def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a text splitter.""" try: - signature = build_template_from_class(name, textsplitter_type_to_cls_dict) - - signature["template"]["documents"] = { - "type": "BaseLoader", - "required": True, - "show": True, - "name": "documents", - } - - signature["template"]["separator"] = { - "type": "str", - "required": True, - "show": True, - "value": ".", - "name": "separator", - "display_name": "Separator", - } - - signature["template"]["chunk_size"] = { - "type": "int", - "required": True, - "show": True, - "value": 4000, - "name": "chunk_size", - "display_name": "Chunk Size", - } - - signature["template"]["chunk_overlap"] = { - "type": "int", - "required": True, - "show": True, - "value": 200, - "name": "chunk_overlap", - "display_name": "Chunk Overlap", - } - - return signature + return build_template_from_class(name, textsplitter_type_to_cls_dict) except ValueError as exc: raise ValueError(f"Text Splitter {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index d00072ad0..8d924f0fe 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -7,7 +7,7 @@ class DocumentLoaderFrontNode(FrontendNode): @staticmethod def build_template(suffixes: list, fileTypes: list, name: str = "file_path" - ) -> Dict: + ) -> TemplateField: """Build a template field for a document loader.""" return TemplateField( field_type="file", @@ -18,68 +18,65 @@ class DocumentLoaderFrontNode(FrontendNode): suffixes=suffixes, fileTypes=fileTypes, ) - - def get_file_path_template(self): - return { - "AirbyteJSONLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": self.build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": self.build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": self.build_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": self.build_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": self.build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": self.build_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredHTMLLoader": self.build_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredImageLoader": self.build_template( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], - ), - "UnstructuredMarkdownLoader": self.build_template( - suffixes=[".md"], fileTypes=["md"] - ), - "PyPDFLoader": self.build_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), - "UnstructuredPowerPointLoader": self.build_template( - suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] - ), - "SRTLoader": self.build_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": self.build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "UnstructuredWordDocumentLoader": self.build_template( - suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] - ), - } + + file_path_templates = { + "AirbyteJSONLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "CoNLLULoader": build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "CSVLoader": build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "UnstructuredEmailLoader": build_template( + suffixes=[".eml"], fileTypes=["eml"] + ), + "EverNoteLoader": build_template( + suffixes=[".xml"], fileTypes=["xml"] + ), + "FacebookChatLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "GutenbergLoader": build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "BSHTMLLoader": build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredHTMLLoader": build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredImageLoader": build_template( + suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + ), + "UnstructuredMarkdownLoader": build_template( + suffixes=[".md"], fileTypes=["md"] + ), + "PyPDFLoader": build_template( + suffixes=[".pdf"], fileTypes=["pdf"] + ), + "UnstructuredPowerPointLoader": build_template( + suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] + ), + "SRTLoader": build_template( + suffixes=[".srt"], fileTypes=["srt"] + ), + "TelegramChatLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "TextLoader": build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "UnstructuredWordDocumentLoader": build_template( + suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] + ), + } def add_extra_fields(self) -> None: - file_path_templates = self.get_file_path_template() - - if self.template.type_name in file_path_templates: - self.template.add_field(file_path_templates[self.template.type_name]) + if self.template.type_name in self.file_path_templates: + self.template.add_field(self.file_path_templates[self.template.type_name]) elif self.template.type_name in { "WebBaseLoader", "AZLyricsLoader", @@ -119,4 +116,6 @@ class DocumentLoaderFrontNode(FrontendNode): value="", display_name="Web Page", ) - ) \ No newline at end of file + ) + + \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index b5f2ce637..49d7d67f3 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -3,7 +3,44 @@ from typing import Dict, List, Optional, Type from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode -class TextSplittersFrontNode(FrontendNode): +class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: - pass \ No newline at end of file + pass + + """ + signature["template"]["documents"] = { + "type": "BaseLoader", + "required": True, + "show": True, + "name": "documents", + } + + signature["template"]["separator"] = { + "type": "str", + "required": True, + "show": True, + "value": ".", + "name": "separator", + "display_name": "Separator", + } + + signature["template"]["chunk_size"] = { + "type": "int", + "required": True, + "show": True, + "value": 4000, + "name": "chunk_size", + "display_name": "Chunk Size", + } + + signature["template"]["chunk_overlap"] = { + "type": "int", + "required": True, + "show": True, + "value": 200, + "name": "chunk_overlap", + "display_name": "Chunk Overlap", + } + """ + \ No newline at end of file From 8aa2d2e4c9c275cb95a50acfbc2825323c431e8e Mon Sep 17 00:00:00 2001 From: Cristhian Zanforlin Lousa Date: Tue, 30 May 2023 19:37:36 -0300 Subject: [PATCH 006/442] Adding Node Toolbar ReactFlow --- .../src/CustomNodes/GenericNode/index.tsx | 349 ++++++++---------- .../components/nodeToolbarComponent/index.tsx | 137 +++++++ 2 files changed, 300 insertions(+), 186 deletions(-) create mode 100644 src/frontend/src/pages/FlowPage/components/nodeToolbarComponent/index.tsx diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index 1a7f93ae5..e8f049f4a 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -30,6 +30,9 @@ import { TabsContext } from "../../contexts/tabsContext"; import { debounce } from "../../utils"; import TooltipReact from "../../components/ReactTooltipComponent"; import Tooltip from "../../components/TooltipComponent"; +import { NodeToolbar } from "reactflow"; +import NodeToolbarComponent from "../../pages/FlowPage/components/nodeToolbarComponent"; + export default function GenericNode({ data, selected, @@ -97,199 +100,173 @@ export default function GenericNode({ deleteNode(data.id); return; } - console.log(data); return ( -
-
-
- -
- -
{data.type}
- -
-
-
- - - - -
- - {validationStatus.params.split("\n").map((line, index) => ( -
{line}
- ))} -
- ) - } > -
- - - -
- -
+ {" "} +
+ {/*
+ Output +
*/} + +
- -
-
- {data.node.description} -
- - <> - {Object.keys(data.node.template) - .filter((t) => t.charAt(0) !== "_") - .map((t: string, idx) => ( -
- {/* {idx === 0 ? ( -
- !key.startsWith("_") && - data.node.template[key].show && - !data.node.template[key].advanced - ).length === 0 - ? "hidden" - : "" - )} - > - Inputs -
- ) : ( - <> - )} */} - {data.node.template[t].show && - !data.node.template[t].advanced ? ( - - ) : ( - <> - )} -
- ))} -
- {" "} -
- {/*
- Output -
*/} - - -
- + ); } diff --git a/src/frontend/src/pages/FlowPage/components/nodeToolbarComponent/index.tsx b/src/frontend/src/pages/FlowPage/components/nodeToolbarComponent/index.tsx new file mode 100644 index 000000000..4720f85ef --- /dev/null +++ b/src/frontend/src/pages/FlowPage/components/nodeToolbarComponent/index.tsx @@ -0,0 +1,137 @@ +import React from "react"; +import { Menu, Transition } from "@headlessui/react"; +import { EllipsisVerticalIcon } from "@heroicons/react/20/solid"; +import { + Cog6ToothIcon, + TrashIcon, + PencilSquareIcon, + DocumentDuplicateIcon, +} from "@heroicons/react/24/outline"; +import { classNames } from "../../../../utils"; +import { Fragment } from "react"; +import NodeModal from "../../../../modals/NodeModal"; + +const NodeToolbarComponent = (props) => { + return ( + <> +
+ + + + + + + + + +
+ + ); +}; + +export default NodeToolbarComponent; From 6cacfced09fd33bbaaf7735c3c97c0e93d364446 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:24:23 -0300 Subject: [PATCH 007/442] =?UTF-8?q?=F0=9F=9A=A7=20chore(conftest.py):=20ad?= =?UTF-8?q?d=20fixtures=20for=20database=20testing=20=E2=9C=85=20test(data?= =?UTF-8?q?base.py):=20add=20tests=20for=20CRUD=20operations=20on=20Flow?= =?UTF-8?q?=20model=20The=20new=20fixtures=20added=20to=20conftest.py=20ar?= =?UTF-8?q?e=20session=20and=20client=20fixtures.=20These=20fixtures=20are?= =?UTF-8?q?=20used=20to=20create=20a=20test=20database=20and=20a=20test=20?= =?UTF-8?q?client=20for=20testing=20the=20database.=20The=20tests=20added?= =?UTF-8?q?=20to=20test=5Fdatabase.py=20test=20the=20CRUD=20operations=20o?= =?UTF-8?q?n=20the=20Flow=20model.=20The=20tests=20include=20creating=20a?= =?UTF-8?q?=20flow,=20reading=20all=20flows,=20reading=20a=20single=20flow?= =?UTF-8?q?,=20updating=20a=20flow,=20and=20deleting=20a=20flow.=20These?= =?UTF-8?q?=20tests=20ensure=20that=20the=20database=20is=20working=20as?= =?UTF-8?q?=20expected=20and=20that=20the=20API=20endpoints=20for=20the=20?= =?UTF-8?q?Flow=20model=20are=20functioning=20correctly.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 35 +++++++++++++++++++++++++ tests/test_database.py | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 tests/test_database.py diff --git a/tests/conftest.py b/tests/conftest.py index 870c48a32..763adf077 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,13 @@ import json from pathlib import Path from typing import AsyncGenerator +from langflow.api.database import get_session import pytest from fastapi.testclient import TestClient from httpx import AsyncClient +from sqlmodel import SQLModel, Session, create_engine +from sqlmodel.pool import StaticPool def pytest_configure(): @@ -76,3 +79,35 @@ def complex_graph(): @pytest.fixture def openapi_graph(): return get_graph("openapi") + + +@pytest.fixture +def json_flow(): + with open(pytest.BASIC_EXAMPLE_PATH, "r") as f: + return f.read() + + +@pytest.fixture(name="session") # +def session_fixture(): # + engine = create_engine( + "sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool + ) + SQLModel.metadata.create_all(engine) + with Session(engine) as session: + yield session + + +@pytest.fixture(name="client") # +def client_fixture(session: Session): # + def get_session_override(): # + return session + + from langflow.main import create_app + + app = create_app() + + app.dependency_overrides[get_session] = get_session_override # + + client = TestClient(app) # + yield client # + app.dependency_overrides.clear() # diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 000000000..247d55e12 --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,58 @@ +from langflow.database.models.flow import FlowCreate + + +def test_create_flow(client, json_flow): + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + assert response.status_code == 200 + assert response.json()["name"] == flow.name + assert response.json()["flow"] == flow.flow + + +def test_read_flows(client, json_flow): + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + assert response.status_code == 200 + assert response.json()["name"] == flow.name + assert response.json()["flow"] == flow.flow + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + assert response.status_code == 200 + assert response.json()["name"] == flow.name + assert response.json()["flow"] == flow.flow + + response = client.get("/flows/") + assert response.status_code == 200 + assert len(response.json()) > 0 + + +def test_read_flow(client, json_flow): + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + flow_id = response.json()["id"] + response = client.get(f"/flows/{flow_id}") + assert response.status_code == 200 + assert response.json()["name"] == flow.name + assert response.json()["flow"] == flow.flow + + +def test_update_flow(client, json_flow): + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + flow_id = response.json()["id"] + updated_flow = FlowCreate( + name="Updated Flow", flow=json_flow.replace("BasicExample", "Updated Flow") + ) + response = client.put(f"/flows/{flow_id}", json=updated_flow.dict()) + assert response.status_code == 200 + assert response.json()["name"] == updated_flow.name + assert response.json()["flow"] == updated_flow.flow + + +def test_delete_flow(client, json_flow): + flow = FlowCreate(name="Test Flow", flow=json_flow) + response = client.post("/flows/", json=flow.dict()) + flow_id = response.json()["id"] + response = client.delete(f"/flows/{flow_id}") + assert response.status_code == 200 + assert response.json()["message"] == "Flow deleted successfully" From 4958e5030a4b5626f7d3fa73c207fed852732573 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:24:45 -0300 Subject: [PATCH 008/442] =?UTF-8?q?=F0=9F=8E=89=20feat(database):=20add=20?= =?UTF-8?q?database=20module=20with=20base=20and=20models=20modules=20The?= =?UTF-8?q?=20database=20module=20has=20been=20added=20to=20the=20project,?= =?UTF-8?q?=20which=20includes=20the=20base=20and=20models=20modules.=20Th?= =?UTF-8?q?e=20base=20module=20contains=20the=20create=5Fdb=5Fand=5Ftables?= =?UTF-8?q?=20function=20that=20creates=20the=20database=20and=20tables,?= =?UTF-8?q?=20and=20the=20get=5Fsession=20function=20that=20returns=20a=20?= =?UTF-8?q?session=20object.=20The=20models=20module=20contains=20the=20Fl?= =?UTF-8?q?ow=20model,=20which=20has=20the=20id,=20name,=20and=20flow=20fi?= =?UTF-8?q?elds.=20The=20FlowBase,=20FlowCreate,=20and=20FlowRead=20classe?= =?UTF-8?q?s=20are=20also=20included=20in=20the=20models=20module.=20These?= =?UTF-8?q?=20classes=20are=20used=20to=20define=20the=20schema=20for=20th?= =?UTF-8?q?e=20Flow=20model.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/database/__init__.py | 0 src/backend/langflow/database/base.py | 17 +++++++++++++++ .../langflow/database/models/__init__.py | 0 src/backend/langflow/database/models/flow.py | 21 +++++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 src/backend/langflow/database/__init__.py create mode 100644 src/backend/langflow/database/base.py create mode 100644 src/backend/langflow/database/models/__init__.py create mode 100644 src/backend/langflow/database/models/flow.py diff --git a/src/backend/langflow/database/__init__.py b/src/backend/langflow/database/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/langflow/database/base.py b/src/backend/langflow/database/base.py new file mode 100644 index 000000000..6aa36baab --- /dev/null +++ b/src/backend/langflow/database/base.py @@ -0,0 +1,17 @@ +from langflow.settings import settings +from sqlmodel import SQLModel, Session, create_engine + +sqlite_file_name = "database.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" + +connect_args = {"check_same_thread": False} +engine = create_engine(sqlite_url, connect_args=connect_args) + + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + + +def get_session(): + with Session(engine) as session: + yield session diff --git a/src/backend/langflow/database/models/__init__.py b/src/backend/langflow/database/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/backend/langflow/database/models/flow.py b/src/backend/langflow/database/models/flow.py new file mode 100644 index 000000000..d7c0ea698 --- /dev/null +++ b/src/backend/langflow/database/models/flow.py @@ -0,0 +1,21 @@ +from typing import List +from pydantic import BaseModel +from sqlmodel import Field, SQLModel +from uuid import UUID, uuid4 + + +class FlowBase(SQLModel): + name: str = Field(index=True) + flow: str = Field(index=False) + + +class Flow(FlowBase, table=True): + id: UUID = Field(default_factory=uuid4, primary_key=True, unique=True) + + +class FlowCreate(FlowBase): + pass + + +class FlowRead(FlowBase): + id: UUID From 8ea2a825a8d0af2ec4b9eb43b8f786bc6b008504 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:25:30 -0300 Subject: [PATCH 009/442] =?UTF-8?q?=F0=9F=9A=80=20feat(api):=20add=20tags?= =?UTF-8?q?=20to=20routers=20for=20better=20organization=20This=20commit?= =?UTF-8?q?=20adds=20tags=20to=20the=20routers=20in=20the=20API=20to=20imp?= =?UTF-8?q?rove=20organization=20and=20make=20it=20easier=20to=20find=20sp?= =?UTF-8?q?ecific=20endpoints.=20The=20Chat=20router=20now=20has=20a=20"Ch?= =?UTF-8?q?at"=20tag,=20the=20Flows=20router=20has=20a=20"Flows"=20tag,=20?= =?UTF-8?q?the=20Validate=20router=20has=20a=20"Validate"=20tag,=20and=20t?= =?UTF-8?q?he=20Base=20router=20has=20a=20"Base"=20tag.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit πŸ› fix(api): fix typo in Validate router prefix This commit fixes a typo in the Validate router prefix. The prefix was misspelled as "validate" instead of "Validate". ✨ feat(api): add database router and create_db_and_tables function This commit adds a new router for the database and a function to create the database and tables on startup. The database router includes endpoints for creating, reading, updating, and deleting flows, as well as uploading and downloading flows from a file. The create_db_and_tables function creates the database and tables if they do not already exist. πŸ› fix(api): fix logger import in main.py This commit fixes an import error in main.py where the logger was being imported from the wrong module. The logger is now imported from langflow.utils.logger. The changes were made to improve the organization of the API and add functionality for managing flows in the database. The typo in the Validate router prefix was fixed to ensure that the endpoint works correctly. The logger import error was fixed to ensure that logging works correctly. --- src/backend/langflow/api/chat.py | 2 +- src/backend/langflow/api/database.py | 103 ++++++++++++++++++++++++++ src/backend/langflow/api/endpoints.py | 16 +++- src/backend/langflow/api/schemas.py | 5 ++ src/backend/langflow/api/validate.py | 2 +- src/backend/langflow/main.py | 12 +++ 6 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 src/backend/langflow/api/database.py diff --git a/src/backend/langflow/api/chat.py b/src/backend/langflow/api/chat.py index 4afa6c22f..9de93abf0 100644 --- a/src/backend/langflow/api/chat.py +++ b/src/backend/langflow/api/chat.py @@ -9,7 +9,7 @@ from fastapi import ( from langflow.api.chat_manager import ChatManager from langflow.utils.logger import logger -router = APIRouter() +router = APIRouter(tags=["Chat"]) chat_manager = ChatManager() diff --git a/src/backend/langflow/api/database.py b/src/backend/langflow/api/database.py new file mode 100644 index 000000000..f6e5cae47 --- /dev/null +++ b/src/backend/langflow/api/database.py @@ -0,0 +1,103 @@ +from typing import List +from uuid import UUID +from langflow.api.schemas import FlowListCreate +from langflow.database.models.flow import Flow, FlowCreate, FlowRead +from langflow.database.base import get_session +from sqlmodel import Session, select +from fastapi import APIRouter, Depends, HTTPException + + +from fastapi import File, UploadFile +import json + +# build router +router = APIRouter(prefix="/flows", tags=["Flows"]) + + +@router.post("/", response_model=FlowRead) +def create_flow(*, session: Session = Depends(get_session), flow: FlowCreate): + """Create a new flow.""" + db_flow = Flow.from_orm(flow) + session.add(db_flow) + session.commit() + session.refresh(db_flow) + return db_flow + + +@router.get("/", response_model=list[FlowRead]) +def read_flows(*, session: Session = Depends(get_session)): + """Read all flows.""" + flows = session.exec(select(Flow)).all() + return flows + + +@router.get("/{flow_id}", response_model=FlowRead) +def read_flow(*, session: Session = Depends(get_session), flow_id: UUID): + """Read a flow.""" + flow = session.get(Flow, flow_id) + if not flow: + raise HTTPException(status_code=404, detail="Flow not found") + return flow + + +@router.put("/{flow_id}", response_model=FlowRead) +def update_flow( + *, session: Session = Depends(get_session), flow_id: UUID, flow: FlowCreate +): + """Update a flow.""" + db_flow = session.get(Flow, flow_id) + if not db_flow: + raise HTTPException(status_code=404, detail="Flow not found") + update_data = flow.dict(exclude_unset=True) + for key, value in update_data.items(): + setattr(db_flow, key, value) + session.add(db_flow) + session.commit() + session.refresh(db_flow) + return db_flow + + +@router.delete("/{flow_id}") +def delete_flow(*, session: Session = Depends(get_session), flow_id: UUID): + """Delete a flow.""" + flow = session.get(Flow, flow_id) + if not flow: + raise HTTPException(status_code=404, detail="Flow not found") + session.delete(flow) + session.commit() + return {"message": "Flow deleted successfully"} + + +# Define a new model to handle multiple flows + + +@router.post("/batch/", response_model=List[FlowRead]) +def create_flows(*, session: Session = Depends(get_session), flow_list: FlowListCreate): + """Create multiple new flows.""" + db_flows = [] + for flow in flow_list.flows: + db_flow = Flow.from_orm(flow) + session.add(db_flow) + db_flows.append(db_flow) + session.commit() + for db_flow in db_flows: + session.refresh(db_flow) + return db_flows + + +@router.post("/upload/", response_model=List[FlowRead]) +async def upload_file( + *, session: Session = Depends(get_session), file: UploadFile = File(...) +): + """Upload flows from a file.""" + contents = await file.read() + data = json.loads(contents) + flow_list = FlowListCreate(**data) + return create_flows(session=session, flow_list=flow_list) + + +@router.get("/download/") +async def download_file(*, session: Session = Depends(get_session)): + """Download all flows as a file.""" + flows = read_flows(session=session) + return {"file": json.dumps([flow.dict() for flow in flows])} diff --git a/src/backend/langflow/api/endpoints.py b/src/backend/langflow/api/endpoints.py index 021a81ca8..dacdad64b 100644 --- a/src/backend/langflow/api/endpoints.py +++ b/src/backend/langflow/api/endpoints.py @@ -1,7 +1,7 @@ -import logging +from langflow.utils.logger import logger from importlib.metadata import version -from fastapi import APIRouter, HTTPException +from fastapi import APIRouter, File, HTTPException, UploadFile from langflow.api.schemas import ( ExportedFlow, @@ -11,10 +11,10 @@ from langflow.api.schemas import ( ) from langflow.interface.run import process_graph_cached from langflow.interface.types import build_langchain_types_dict +from langflow.cache import cache_manager # build router -router = APIRouter() -logger = logging.getLogger(__name__) +router = APIRouter(tags=["Base"]) @router.get("/all") @@ -45,3 +45,11 @@ def get_version(): @router.get("/health") def get_health(): return {"status": "OK"} + + +# Make an endpoint to upload a file using the client_id and +# cache the file in the backend +@router.post("/uploadfile/{client_id}") +async def create_upload_file(client_id: str, file: UploadFile = File(...)): + + # TODO: Implement this endpoint diff --git a/src/backend/langflow/api/schemas.py b/src/backend/langflow/api/schemas.py index f73b0642d..a9cb4dcb6 100644 --- a/src/backend/langflow/api/schemas.py +++ b/src/backend/langflow/api/schemas.py @@ -1,4 +1,5 @@ from typing import Any, Dict, List, Union +from langflow.database.models.flow import FlowCreate from pydantic import BaseModel, validator @@ -68,3 +69,7 @@ class FileResponse(ChatMessage): if v not in ["image", "csv"]: raise ValueError("data_type must be image or csv") return v + + +class FlowListCreate(BaseModel): + flows: List[FlowCreate] diff --git a/src/backend/langflow/api/validate.py b/src/backend/langflow/api/validate.py index 0e2a7752c..fade4bdf8 100644 --- a/src/backend/langflow/api/validate.py +++ b/src/backend/langflow/api/validate.py @@ -15,7 +15,7 @@ from langflow.utils.logger import logger from langflow.utils.validate import validate_code # build router -router = APIRouter(prefix="/validate", tags=["validate"]) +router = APIRouter(prefix="/validate", tags=["Validate"]) @router.post("/code", status_code=200, response_model=CodeValidationResponse) diff --git a/src/backend/langflow/main.py b/src/backend/langflow/main.py index 56cc32e46..75c19bd25 100644 --- a/src/backend/langflow/main.py +++ b/src/backend/langflow/main.py @@ -4,6 +4,14 @@ from fastapi.middleware.cors import CORSMiddleware from langflow.api.chat import router as chat_router from langflow.api.endpoints import router as endpoints_router from langflow.api.validate import router as validate_router +from langflow.api.database import router as database_router +from langflow.utils.logger import logger +from langflow.database.base import create_db_and_tables +from fastapi import APIRouter, HTTPException + + +# build router +router = APIRouter() def create_app(): @@ -25,6 +33,10 @@ def create_app(): app.include_router(endpoints_router) app.include_router(validate_router) app.include_router(chat_router) + app.include_router(database_router) + + app.on_event("startup")(create_db_and_tables) + return app From bcd77a641d188848a7d49ead5860f17783b0e5cb Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:25:48 -0300 Subject: [PATCH 010/442] =?UTF-8?q?=F0=9F=93=A6=20add(pyproject.toml):=20a?= =?UTF-8?q?dd=20sqlmodel=20dependency=20to=20dev=20dependencies=20The=20sq?= =?UTF-8?q?lmodel=20dependency=20has=20been=20added=20to=20the=20dev=20dep?= =?UTF-8?q?endencies=20section=20of=20the=20pyproject.toml=20file.=20This?= =?UTF-8?q?=20is=20to=20enable=20the=20use=20of=20sqlmodel=20in=20developm?= =?UTF-8?q?ent=20and=20testing=20of=20the=20application.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- poetry.lock | 139 ++++++++++++++++++++++++++++++------------------- pyproject.toml | 1 + 2 files changed, 85 insertions(+), 55 deletions(-) diff --git a/poetry.lock b/poetry.lock index a414aab16..b2b3a9902 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4951,82 +4951,111 @@ files = [ [[package]] name = "sqlalchemy" -version = "2.0.15" +version = "1.4.41" description = "Database Abstraction Library" category = "main" optional = false -python-versions = ">=3.7" +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "SQLAlchemy-2.0.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:78303719c6f72af97814b0072ad18bee72e70adca8d95cf8fecd59c5e1ddb040"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9d810b4aacd5ef4e293aa4ea01f19fca53999e9edcfc4a8ef1146238b30bdc28"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fb5d09f1d51480f711b69fe28ad42e4f8b08600a85ab2473baee669e1257800"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51b19887c96d405599880da6a7cbdf8545a7e78ec5683e46a43bac8885e32d0f"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d6b17cb86908e7f88be14007d6afe7d2ab11966e373044137f96a6a4d83eb21c"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df25052b92bd514357a9b370d74f240db890ea79aaa428fb893520e10ee5bc18"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-win32.whl", hash = "sha256:55ec62ddc0200b4fee94d11abbec7aa25948d5d21cb8df8807f4bdd3c51ba44b"}, - {file = "SQLAlchemy-2.0.15-cp310-cp310-win_amd64.whl", hash = "sha256:ae1d8deb391ab39cc8f0d5844e588a115ae3717e607d91482023917f920f777f"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4670ce853cb25f72115a1bbe366ae13cf3f28fc5c87222df14f8d3d55d51816e"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cea7c4a3dfc2ca61f88a2b1ddd6b0bfbd116c9b1a361b3b66fd826034b833142"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f5784dfb2d45c19cde03c45c04a54bf47428610106197ed6e6fa79f33bc63d3"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b31ebde27575b3b0708673ec14f0c305c4564d995b545148ab7ac0f4d9b847a"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6b42913a0259267e9ee335da0c36498077799e59c5e332d506e72b4f32de781d"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a3f8020e013e9b3b7941dcf20b0fc8f7429daaf7158760846731cbd8caa5e45"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-win32.whl", hash = "sha256:88ab245ed2c96265441ed2818977be28c840cfa5204ba167425d6c26eb67b7e7"}, - {file = "SQLAlchemy-2.0.15-cp311-cp311-win_amd64.whl", hash = "sha256:5cc48a7fda2b5c5b8860494d6c575db3a101a68416492105fed6591dc8a2728a"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f6fd3c88ea4b170d13527e93be1945e69facd917661d3725a63470eb683fbffe"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e885dacb167077df15af2f9ccdacbd7f5dd0d538a6d74b94074f2cefc7bb589"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:201a99f922ac8c780b3929128fbd9df901418877c70e160e19adb05665e51c31"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:e17fdcb8971e77c439113642ca8861f9465e21fc693bd3916654ceef3ac26883"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db269f67ed17b07e80aaa8fba1f650c0d84aa0bdd9d5352e4ac38d5bf47ac568"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-win32.whl", hash = "sha256:994a75b197662e0608b6a76935d7c345f7fd874eac0b7093d561033db61b0e8c"}, - {file = "SQLAlchemy-2.0.15-cp37-cp37m-win_amd64.whl", hash = "sha256:4d61731a35eddb0f667774fe15e5a4831e444d066081d1e809e1b8a0e3f97cae"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f7f994a53c0e6b44a2966fd6bfc53e37d34b7dca34e75b6be295de6db598255e"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:79bfe728219239bdc493950ea4a4d15b02138ecb304771f9024d0d6f5f4e3706"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d6320a1d175447dce63618ec997a53836de48ed3b44bbe952f0b4b399b19941"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f80a9c9a9af0e4bd5080cc0955ce70274c28e9b931ad7e0fb07021afcd32af6"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4a75fdb9a84072521bb2ebd31eefe1165d4dccea3039dda701a864f4b5daa17f"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:21c89044fc48a25c2184eba332edeffbbf9367913bb065cd31538235d828f06f"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-win32.whl", hash = "sha256:1a0754c2d9f0c7982bec0a31138e495ed1f6b8435d7e677c45be60ec18370acf"}, - {file = "SQLAlchemy-2.0.15-cp38-cp38-win_amd64.whl", hash = "sha256:bc5c2b0da46c26c5f73f700834f871d0723e1e882641932468d56833bab09775"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:670ecf74ee2e70b917028a06446ad26ff9b1195e84b09c3139c215123d57dc30"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d14282bf5b4de87f922db3c70858953fd081ef4f05dba6cca3dd705daffe1cc9"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:256b2b9660e51ad7055a9835b12717416cf7288afcf465107413917b6bb2316f"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810199d1c5b43603a9e815ae9487aef3ab1ade7ed9c0c485e12519358929fbfe"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:536c86ec81ca89291d533ff41a3a05f9e4e88e01906dcee0751fc7082f3e8d6c"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:435f6807fa6a0597d84741470f19db204a7d34625ea121abd63e8d95f673f0c4"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-win32.whl", hash = "sha256:da7381a883aee20b7d2ffda17d909b38134b6a625920e65239a1c681881df800"}, - {file = "SQLAlchemy-2.0.15-cp39-cp39-win_amd64.whl", hash = "sha256:788d1772fb8dcd12091ca82809eef504ce0f2c423e45284bc351b872966ff554"}, - {file = "SQLAlchemy-2.0.15-py3-none-any.whl", hash = "sha256:933d30273861fe61f014ce2a7e3c364915f5efe9ed250ec1066ca6ea5942c0bd"}, - {file = "SQLAlchemy-2.0.15.tar.gz", hash = "sha256:2e940a8659ef870ae10e0d9e2a6d5aaddf0ff6e91f7d0d7732afc9e8c4be9bbc"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:13e397a9371ecd25573a7b90bd037db604331cf403f5318038c46ee44908c44d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:2d6495f84c4fd11584f34e62f9feec81bf373787b3942270487074e35cbe5330"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win32.whl", hash = "sha256:e570cfc40a29d6ad46c9aeaddbdcee687880940a3a327f2c668dd0e4ef0a441d"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27m-win_amd64.whl", hash = "sha256:5facb7fd6fa8a7353bbe88b95695e555338fb038ad19ceb29c82d94f62775a05"}, + {file = "SQLAlchemy-1.4.41-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:f37fa70d95658763254941ddd30ecb23fc4ec0c5a788a7c21034fc2305dab7cc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:361f6b5e3f659e3c56ea3518cf85fbdae1b9e788ade0219a67eeaaea8a4e4d2a"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0990932f7cca97fece8017414f57fdd80db506a045869d7ddf2dda1d7cf69ecc"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cd767cf5d7252b1c88fcfb58426a32d7bd14a7e4942497e15b68ff5d822b41ad"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5102fb9ee2c258a2218281adcb3e1918b793c51d6c2b4666ce38c35101bb940e"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win32.whl", hash = "sha256:2082a2d2fca363a3ce21cfa3d068c5a1ce4bf720cf6497fb3a9fc643a8ee4ddd"}, + {file = "SQLAlchemy-1.4.41-cp310-cp310-win_amd64.whl", hash = "sha256:e4b12e3d88a8fffd0b4ca559f6d4957ed91bd4c0613a4e13846ab8729dc5c251"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:90484a2b00baedad361402c257895b13faa3f01780f18f4a104a2f5c413e4536"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b67fc780cfe2b306180e56daaa411dd3186bf979d50a6a7c2a5b5036575cbdbb"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ad2b727fc41c7f8757098903f85fafb4bf587ca6605f82d9bf5604bd9c7cded"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win32.whl", hash = "sha256:59bdc291165b6119fc6cdbc287c36f7f2859e6051dd923bdf47b4c55fd2f8bd0"}, + {file = "SQLAlchemy-1.4.41-cp311-cp311-win_amd64.whl", hash = "sha256:d2e054aed4645f9b755db85bc69fc4ed2c9020c19c8027976f66576b906a74f1"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:4ba7e122510bbc07258dc42be6ed45997efdf38129bde3e3f12649be70683546"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0dcf127bb99458a9d211e6e1f0f3edb96c874dd12f2503d4d8e4f1fd103790b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e16c2be5cb19e2c08da7bd3a87fed2a0d4e90065ee553a940c4fc1a0fb1ab72b"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5ebeeec5c14533221eb30bad716bc1fd32f509196318fb9caa7002c4a364e4c"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win32.whl", hash = "sha256:3e2ef592ac3693c65210f8b53d0edcf9f4405925adcfc031ff495e8d18169682"}, + {file = "SQLAlchemy-1.4.41-cp36-cp36m-win_amd64.whl", hash = "sha256:eb30cf008850c0a26b72bd1b9be6730830165ce049d239cfdccd906f2685f892"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:c23d64a0b28fc78c96289ffbd0d9d1abd48d267269b27f2d34e430ea73ce4b26"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8eb8897367a21b578b26f5713833836f886817ee2ffba1177d446fa3f77e67c8"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:14576238a5f89bcf504c5f0a388d0ca78df61fb42cb2af0efe239dc965d4f5c9"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:639e1ae8d48b3c86ffe59c0daa9a02e2bfe17ca3d2b41611b30a0073937d4497"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win32.whl", hash = "sha256:0005bd73026cd239fc1e8ccdf54db58b6193be9a02b3f0c5983808f84862c767"}, + {file = "SQLAlchemy-1.4.41-cp37-cp37m-win_amd64.whl", hash = "sha256:5323252be2bd261e0aa3f33cb3a64c45d76829989fa3ce90652838397d84197d"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:05f0de3a1dc3810a776275763764bb0015a02ae0f698a794646ebc5fb06fad33"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0002e829142b2af00b4eaa26c51728f3ea68235f232a2e72a9508a3116bd6ed0"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:22ff16cedab5b16a0db79f1bc99e46a6ddececb60c396562e50aab58ddb2871c"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccfd238f766a5bb5ee5545a62dd03f316ac67966a6a658efb63eeff8158a4bbf"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win32.whl", hash = "sha256:58bb65b3274b0c8a02cea9f91d6f44d0da79abc993b33bdedbfec98c8440175a"}, + {file = "SQLAlchemy-1.4.41-cp38-cp38-win_amd64.whl", hash = "sha256:ce8feaa52c1640de9541eeaaa8b5fb632d9d66249c947bb0d89dd01f87c7c288"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:199a73c31ac8ea59937cc0bf3dfc04392e81afe2ec8a74f26f489d268867846c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4676d51c9f6f6226ae8f26dc83ec291c088fe7633269757d333978df78d931ab"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:036d8472356e1d5f096c5e0e1a7e0f9182140ada3602f8fff6b7329e9e7cfbcd"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2307495d9e0ea00d0c726be97a5b96615035854972cc538f6e7eaed23a35886c"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win32.whl", hash = "sha256:9c56e19780cd1344fcd362fd6265a15f48aa8d365996a37fab1495cae8fcd97d"}, + {file = "SQLAlchemy-1.4.41-cp39-cp39-win_amd64.whl", hash = "sha256:f5fa526d027d804b1f85cdda1eb091f70bde6fb7d87892f6dd5a48925bc88898"}, + {file = "SQLAlchemy-1.4.41.tar.gz", hash = "sha256:0292f70d1797e3c54e862e6f30ae474014648bc9c723e14a2fda730adb0a9791"}, ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} -typing-extensions = ">=4.2.0" +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and platform_machine == \"aarch64\" or python_version >= \"3\" and platform_machine == \"ppc64le\" or python_version >= \"3\" and platform_machine == \"x86_64\" or python_version >= \"3\" and platform_machine == \"amd64\" or python_version >= \"3\" and platform_machine == \"AMD64\" or python_version >= \"3\" and platform_machine == \"win32\" or python_version >= \"3\" and platform_machine == \"WIN32\""} [package.extras] aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)"] -mysql = ["mysqlclient (>=1.4.0)"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)"] -oracle-oracledb = ["oracledb (>=1.0.1)"] +oracle = ["cx-oracle (>=7)", "cx-oracle (>=7,<8)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.29.1)"] -postgresql-psycopg = ["psycopg (>=3.0.7)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql"] +pymysql = ["pymysql", "pymysql (<1)"] sqlcipher = ["sqlcipher3-binary"] +[[package]] +name = "sqlalchemy2-stubs" +version = "0.0.2a34" +description = "Typing Stubs for SQLAlchemy 1.4" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "sqlalchemy2-stubs-0.0.2a34.tar.gz", hash = "sha256:2432137ab2fde1a608df4544f6712427b0b7ff25990cfbbc5a9d1db6c8c6f489"}, + {file = "sqlalchemy2_stubs-0.0.2a34-py3-none-any.whl", hash = "sha256:a313220ac793404349899faf1272e821a62dbe1d3a029bd444faa8d3e966cd07"}, +] + +[package.dependencies] +typing-extensions = ">=3.7.4" + +[[package]] +name = "sqlmodel" +version = "0.0.8" +description = "SQLModel, SQL databases in Python, designed for simplicity, compatibility, and robustness." +category = "main" +optional = false +python-versions = ">=3.6.1,<4.0.0" +files = [ + {file = "sqlmodel-0.0.8-py3-none-any.whl", hash = "sha256:0fd805719e0c5d4f22be32eb3ffc856eca3f7f20e8c7aa3e117ad91684b518ee"}, + {file = "sqlmodel-0.0.8.tar.gz", hash = "sha256:3371b4d1ad59d2ffd0c530582c2140b6c06b090b32af9b9c6412986d7b117036"}, +] + +[package.dependencies] +pydantic = ">=1.8.2,<2.0.0" +SQLAlchemy = ">=1.4.17,<=1.4.41" +sqlalchemy2-stubs = "*" + [[package]] name = "stack-data" version = "0.6.2" @@ -6178,4 +6207,4 @@ deploy = ["langchain-serve"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "9ce165d2decf2190d7ce69be608872b3ed9abe705a276045623706d01665754b" +content-hash = "a293ac634ddc277680a1293b396f84edd897946d92b04bdff9f35b14c935a8f8" diff --git a/pyproject.toml b/pyproject.toml index 79d09b422..ff91ae43e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ jina = "3.15.2" sentence-transformers = "^2.2.2" ctransformers = "^0.2.2" cohere = "^4.6.0" +sqlmodel = "^0.0.8" [tool.poetry.group.dev.dependencies] From 4b6a8595df8e519a24fe1c064baf85c04ebe345b Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:34:23 -0300 Subject: [PATCH 011/442] =?UTF-8?q?=F0=9F=90=9B=20fix(nodes.py):=20change?= =?UTF-8?q?=20type=20hint=20of=20tools=20list=20to=20include=20ToolkitNode?= =?UTF-8?q?=20The=20type=20hint=20of=20the=20tools=20list=20in=20the=20Age?= =?UTF-8?q?ntNode=20class=20has=20been=20updated=20to=20include=20the=20To?= =?UTF-8?q?olkitNode=20class.=20This=20is=20because=20the=20tools=20list?= =?UTF-8?q?=20can=20now=20contain=20instances=20of=20the=20ToolkitNode=20c?= =?UTF-8?q?lass=20in=20addition=20to=20the=20ToolNode=20class.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/graph/nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/graph/nodes.py b/src/backend/langflow/graph/nodes.py index 5f136b3c7..9db6260e9 100644 --- a/src/backend/langflow/graph/nodes.py +++ b/src/backend/langflow/graph/nodes.py @@ -8,7 +8,7 @@ class AgentNode(Node): def __init__(self, data: Dict): super().__init__(data, base_type="agents") - self.tools: List[ToolNode] = [] + self.tools: List[Union[ToolNode, ToolkitNode]] = [] self.chains: List[ChainNode] = [] def _set_tools_and_chains(self) -> None: From 041748b2fb29a1641e889973baddf56941611fe9 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 21:47:42 -0300 Subject: [PATCH 012/442] =?UTF-8?q?=F0=9F=94=A8=20refactor(nodes.py):=20ex?= =?UTF-8?q?tract=20flatten=5Flist=20function=20to=20utils=20module=20and?= =?UTF-8?q?=20use=20it=20in=20PromptNode.build=20method=20=F0=9F=90=9B=20f?= =?UTF-8?q?ix(nodes.py):=20change=20tools=20parameter=20type=20hint=20in?= =?UTF-8?q?=20PromptNode.build=20method=20to=20accept=20a=20list=20of=20Un?= =?UTF-8?q?ion[ToolNode,=20ToolkitNode]=20The=20flatten=5Flist=20function?= =?UTF-8?q?=20was=20extracted=20from=20the=20PromptNode.build=20method=20a?= =?UTF-8?q?nd=20moved=20to=20the=20utils=20module=20to=20improve=20code=20?= =?UTF-8?q?reusability.=20The=20PromptNode.build=20method=20now=20uses=20t?= =?UTF-8?q?he=20flatten=5Flist=20function=20to=20flatten=20the=20list=20of?= =?UTF-8?q?=20tools=20if=20it=20is=20a=20list=20of=20lists.=20The=20tools?= =?UTF-8?q?=20parameter=20type=20hint=20was=20changed=20to=20accept=20a=20?= =?UTF-8?q?list=20of=20Union[ToolNode,=20ToolkitNode]=20to=20improve=20typ?= =?UTF-8?q?e=20safety.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/graph/nodes.py | 62 ++++++++++++++--------------- src/backend/langflow/graph/utils.py | 12 ++++++ 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/backend/langflow/graph/nodes.py b/src/backend/langflow/graph/nodes.py index 9db6260e9..21fe0f673 100644 --- a/src/backend/langflow/graph/nodes.py +++ b/src/backend/langflow/graph/nodes.py @@ -1,7 +1,12 @@ from typing import Any, Dict, List, Optional, Union from langflow.graph.base import Node -from langflow.graph.utils import extract_input_variables_from_prompt +from langflow.graph.utils import extract_input_variables_from_prompt, flatten_list + + +class ToolkitNode(Node): + def __init__(self, data: Dict): + super().__init__(data, base_type="toolkits") class AgentNode(Node): @@ -47,7 +52,7 @@ class PromptNode(Node): def build( self, force: bool = False, - tools: Optional[Union[List[Node], List[ToolNode]]] = None, + tools: Optional[List[Union[ToolNode, ToolkitNode]]] = None, ) -> Any: if not self._built or force: if ( @@ -65,8 +70,7 @@ class PromptNode(Node): # flatten the list of tools if it is a list of lists # first check if it is a list if tools and isinstance(tools, list) and isinstance(tools[0], list): - tools = [tool for sublist in tools for tool in sublist] - + tools = flatten_list(tools) self.params["tools"] = tools prompt_params = [ key @@ -85,30 +89,6 @@ class PromptNode(Node): return self._built_object -class ChainNode(Node): - def __init__(self, data: Dict): - super().__init__(data, base_type="chains") - - def build( - self, - force: bool = False, - tools: Optional[Union[List[Node], List[ToolNode]]] = None, - ) -> Any: - if not self._built or force: - # Check if the chain requires a PromptNode - for key, value in self.params.items(): - if isinstance(value, PromptNode): - # Build the PromptNode, passing the tools if available - self.params[key] = value.build(tools=tools, force=force) - - self._build() - - #! Cannot deepcopy SQLDatabaseChain - if self.node_type in ["SQLDatabaseChain"]: - return self._built_object - return self._built_object - - class LLMNode(Node): built_node_type = None class_built_object = None @@ -130,11 +110,6 @@ class LLMNode(Node): return self._built_object -class ToolkitNode(Node): - def __init__(self, data: Dict): - super().__init__(data, base_type="toolkits") - - class FileToolNode(ToolNode): def __init__(self, data: Dict): super().__init__(data) @@ -193,3 +168,24 @@ class TextSplitterNode(Node): if self._built_object: return f"""{self.node_type}({len(self._built_object)} documents)\nDocuments: {self._built_object[:3]}...""" return f"{self.node_type}()" + + +class ChainNode(Node): + def __init__(self, data: Dict): + super().__init__(data, base_type="chains") + + def build( + self, + force: bool = False, + tools: Optional[List[Union[ToolNode, ToolkitNode]]] = None, + ) -> Any: + if not self._built or force: + # Check if the chain requires a PromptNode + for key, value in self.params.items(): + if isinstance(value, PromptNode): + # Build the PromptNode, passing the tools if available + self.params[key] = value.build(tools=tools, force=force) + + self._build() + + return self._built_object diff --git a/src/backend/langflow/graph/utils.py b/src/backend/langflow/graph/utils.py index 6d56e933e..e22b27cf5 100644 --- a/src/backend/langflow/graph/utils.py +++ b/src/backend/langflow/graph/utils.py @@ -1,4 +1,5 @@ import re +from typing import Any, Union def validate_prompt(prompt: str): @@ -17,3 +18,14 @@ def fix_prompt(prompt: str): def extract_input_variables_from_prompt(prompt: str) -> list[str]: """Extract input variables from prompt.""" return re.findall(r"{(.*?)}", prompt) + + +def flatten_list(list_of_lists: list[Union[list, Any]]) -> list: + """Flatten list of lists.""" + new_list = [] + for item in list_of_lists: + if isinstance(item, list): + new_list.extend(item) + else: + new_list.append(item) + return new_list From 4160274b298a8ade902eccecc36268a80bb849dd Mon Sep 17 00:00:00 2001 From: Cristhian Zanforlin Lousa Date: Tue, 30 May 2023 22:15:01 -0300 Subject: [PATCH 013/442] Adding search nodes to ExtraSideBarComponent --- .../ExtraSidebarComponent/index.tsx | 9 +- .../extraSidebarComponent/index.tsx | 163 +++++++++++------- 2 files changed, 107 insertions(+), 65 deletions(-) diff --git a/src/frontend/src/components/ExtraSidebarComponent/index.tsx b/src/frontend/src/components/ExtraSidebarComponent/index.tsx index f5acd5a23..b06a58d9c 100644 --- a/src/frontend/src/components/ExtraSidebarComponent/index.tsx +++ b/src/frontend/src/components/ExtraSidebarComponent/index.tsx @@ -1,6 +1,6 @@ import { Disclosure } from "@headlessui/react"; import { ChevronLeftIcon } from "@heroicons/react/24/outline"; -import { useContext } from "react"; +import { useContext, useState } from "react"; import { Link } from "react-router-dom"; import { classNames } from "../../utils"; import { locationContext } from "../../contexts/locationContext"; @@ -13,6 +13,7 @@ export default function ExtraSidebar() { extraNavigation, extraComponent, } = useContext(locationContext); + return ( <>