From 7edce61182473a647b4b549a1d389b2d379531b5 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Tue, 30 May 2023 17:58:13 -0300 Subject: [PATCH 01/69] Implemented add_extra_fields method for the documentloaders --- .../interface/document_loaders/base.py | 132 +++--------------- .../template/frontend_node/__init__.py | 4 + .../template/frontend_node/documentloaders.py | 122 ++++++++++++++++ .../template/frontend_node/textsplitters.py | 9 ++ .../template/frontend_node/vectorstores.py | 1 - 5 files changed, 156 insertions(+), 112 deletions(-) create mode 100644 src/backend/langflow/template/frontend_node/documentloaders.py create mode 100644 src/backend/langflow/template/frontend_node/textsplitters.py diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index a13d5cd5b..484db584c 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -1,134 +1,44 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode from langflow.interface.custom_lists import documentloaders_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class - def build_file_path_template( - suffixes: list, fileTypes: list, name: str = "file_path" -) -> Dict: - """Build a file path template for a document loader.""" - return { - "type": "file", - "required": True, - "show": True, - "name": name, - "value": "", - "suffixes": suffixes, - "fileTypes": fileTypes, - } - + suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a file path template for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" + @property + def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]: + return DocumentLoaderFrontNode + @property def type_to_loader_dict(self) -> Dict: return documentloaders_type_to_cls_dict - + def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a document loader.""" try: - signature = build_template_from_class( + return build_template_from_class( name, documentloaders_type_to_cls_dict ) - - file_path_templates = { - "AirbyteJSONLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": build_file_path_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": build_file_path_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredImageLoader": build_file_path_template( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], - ), - "UnstructuredMarkdownLoader": build_file_path_template( - suffixes=[".md"], fileTypes=["md"] - ), - "PyPDFLoader": build_file_path_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), - "UnstructuredPowerPointLoader": build_file_path_template( - suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] - ), - "SRTLoader": build_file_path_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "UnstructuredWordDocumentLoader": build_file_path_template( - suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] - ), - } - - if name in file_path_templates: - signature["template"]["file_path"] = file_path_templates[name] - elif name in { - "WebBaseLoader", - "AZLyricsLoader", - "CollegeConfidentialLoader", - "HNLoader", - "IFixitLoader", - "IMSDbLoader", - }: - signature["template"]["web_path"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_path", - "value": "", - "display_name": "Web Page", - } - elif name in {"GitbookLoader"}: - signature["template"]["web_page"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_page", - "value": "", - "display_name": "Web Page", - } - elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}: - signature["template"]["path"] = { - "type": "str", - "required": True, - "show": True, - "name": "path", - "value": "", - "display_name": "Web Page", - } - - return signature except ValueError as exc: raise ValueError(f"Documment Loader {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/template/frontend_node/__init__.py b/src/backend/langflow/template/frontend_node/__init__.py index 1aa946d41..c36234364 100644 --- a/src/backend/langflow/template/frontend_node/__init__.py +++ b/src/backend/langflow/template/frontend_node/__init__.py @@ -7,6 +7,8 @@ from langflow.template.frontend_node import ( prompts, tools, vectorstores, + documentloaders, + textsplitters, ) __all__ = [ @@ -18,4 +20,6 @@ __all__ = [ "llms", "prompts", "vectorstores", + "documentloaders", + "textsplitters", ] diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py new file mode 100644 index 000000000..d00072ad0 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -0,0 +1,122 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class DocumentLoaderFrontNode(FrontendNode): + + @staticmethod + def build_template(suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a template field for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) + + def get_file_path_template(self): + return { + "AirbyteJSONLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "CoNLLULoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "CSVLoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "UnstructuredEmailLoader": self.build_template( + suffixes=[".eml"], fileTypes=["eml"] + ), + "EverNoteLoader": self.build_template( + suffixes=[".xml"], fileTypes=["xml"] + ), + "FacebookChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "GutenbergLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "BSHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredImageLoader": self.build_template( + suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + ), + "UnstructuredMarkdownLoader": self.build_template( + suffixes=[".md"], fileTypes=["md"] + ), + "PyPDFLoader": self.build_template( + suffixes=[".pdf"], fileTypes=["pdf"] + ), + "UnstructuredPowerPointLoader": self.build_template( + suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] + ), + "SRTLoader": self.build_template( + suffixes=[".srt"], fileTypes=["srt"] + ), + "TelegramChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "TextLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "UnstructuredWordDocumentLoader": self.build_template( + suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] + ), + } + + def add_extra_fields(self) -> None: + file_path_templates = self.get_file_path_template() + + if self.template.type_name in file_path_templates: + self.template.add_field(file_path_templates[self.template.type_name]) + elif self.template.type_name in { + "WebBaseLoader", + "AZLyricsLoader", + "CollegeConfidentialLoader", + "HNLoader", + "IFixitLoader", + "IMSDbLoader", + }: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_path", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"GitbookLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_page", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"ReadTheDocsLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="path", + value="", + display_name="Web Page", + ) + ) \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py new file mode 100644 index 000000000..b5f2ce637 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -0,0 +1,9 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class TextSplittersFrontNode(FrontendNode): + + def add_extra_fields(self) -> None: + pass \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index 1aefaf10c..e071c3d4c 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -3,7 +3,6 @@ from typing import Optional from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode - class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: if self.template.type_name == "Weaviate": From 5291e981e2340e5bf36c7c713665411692d3a97f Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Tue, 30 May 2023 19:22:59 -0300 Subject: [PATCH 02/69] Implemented add_extra_fields for the documentloaders --- .../interface/document_loaders/base.py | 14 -- .../langflow/interface/text_splitters/base.py | 45 +------ .../template/frontend_node/documentloaders.py | 123 +++++++++--------- .../template/frontend_node/textsplitters.py | 41 +++++- 4 files changed, 107 insertions(+), 116 deletions(-) diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index 484db584c..0a0efbb71 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -8,20 +8,6 @@ from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class -def build_file_path_template( - suffixes: list, fileTypes: list, name: str = "file_path" - ) -> Dict: - """Build a file path template for a document loader.""" - return TemplateField( - field_type="file", - required=True, - show=True, - name=name, - value="", - suffixes=suffixes, - fileTypes=fileTypes, - ) - class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" diff --git a/src/backend/langflow/interface/text_splitters/base.py b/src/backend/langflow/interface/text_splitters/base.py index e58a5bfa6..11dfaca88 100644 --- a/src/backend/langflow/interface/text_splitters/base.py +++ b/src/backend/langflow/interface/text_splitters/base.py @@ -1,6 +1,7 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator +from langflow.template.frontend_node.textsplitters import VectorStoreFrontendNode from langflow.interface.custom_lists import textsplitter_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger @@ -10,6 +11,10 @@ from langflow.utils.util import build_template_from_class class TextSplitterCreator(LangChainTypeCreator): type_name: str = "textsplitters" + @property + def frontend_node_class(self) -> Type[VectorStoreFrontendNode]: + return VectorStoreFrontendNode + @property def type_to_loader_dict(self) -> Dict: return textsplitter_type_to_cls_dict @@ -17,43 +22,7 @@ class TextSplitterCreator(LangChainTypeCreator): def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a text splitter.""" try: - signature = build_template_from_class(name, textsplitter_type_to_cls_dict) - - signature["template"]["documents"] = { - "type": "BaseLoader", - "required": True, - "show": True, - "name": "documents", - } - - signature["template"]["separator"] = { - "type": "str", - "required": True, - "show": True, - "value": ".", - "name": "separator", - "display_name": "Separator", - } - - signature["template"]["chunk_size"] = { - "type": "int", - "required": True, - "show": True, - "value": 4000, - "name": "chunk_size", - "display_name": "Chunk Size", - } - - signature["template"]["chunk_overlap"] = { - "type": "int", - "required": True, - "show": True, - "value": 200, - "name": "chunk_overlap", - "display_name": "Chunk Overlap", - } - - return signature + return build_template_from_class(name, textsplitter_type_to_cls_dict) except ValueError as exc: raise ValueError(f"Text Splitter {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index d00072ad0..8d924f0fe 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -7,7 +7,7 @@ class DocumentLoaderFrontNode(FrontendNode): @staticmethod def build_template(suffixes: list, fileTypes: list, name: str = "file_path" - ) -> Dict: + ) -> TemplateField: """Build a template field for a document loader.""" return TemplateField( field_type="file", @@ -18,68 +18,65 @@ class DocumentLoaderFrontNode(FrontendNode): suffixes=suffixes, fileTypes=fileTypes, ) - - def get_file_path_template(self): - return { - "AirbyteJSONLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": self.build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": self.build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": self.build_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": self.build_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": self.build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": self.build_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredHTMLLoader": self.build_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredImageLoader": self.build_template( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], - ), - "UnstructuredMarkdownLoader": self.build_template( - suffixes=[".md"], fileTypes=["md"] - ), - "PyPDFLoader": self.build_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), - "UnstructuredPowerPointLoader": self.build_template( - suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] - ), - "SRTLoader": self.build_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": self.build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": self.build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "UnstructuredWordDocumentLoader": self.build_template( - suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] - ), - } + + file_path_templates = { + "AirbyteJSONLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "CoNLLULoader": build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "CSVLoader": build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "UnstructuredEmailLoader": build_template( + suffixes=[".eml"], fileTypes=["eml"] + ), + "EverNoteLoader": build_template( + suffixes=[".xml"], fileTypes=["xml"] + ), + "FacebookChatLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "GutenbergLoader": build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "BSHTMLLoader": build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredHTMLLoader": build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredImageLoader": build_template( + suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + ), + "UnstructuredMarkdownLoader": build_template( + suffixes=[".md"], fileTypes=["md"] + ), + "PyPDFLoader": build_template( + suffixes=[".pdf"], fileTypes=["pdf"] + ), + "UnstructuredPowerPointLoader": build_template( + suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] + ), + "SRTLoader": build_template( + suffixes=[".srt"], fileTypes=["srt"] + ), + "TelegramChatLoader": build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "TextLoader": build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "UnstructuredWordDocumentLoader": build_template( + suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] + ), + } def add_extra_fields(self) -> None: - file_path_templates = self.get_file_path_template() - - if self.template.type_name in file_path_templates: - self.template.add_field(file_path_templates[self.template.type_name]) + if self.template.type_name in self.file_path_templates: + self.template.add_field(self.file_path_templates[self.template.type_name]) elif self.template.type_name in { "WebBaseLoader", "AZLyricsLoader", @@ -119,4 +116,6 @@ class DocumentLoaderFrontNode(FrontendNode): value="", display_name="Web Page", ) - ) \ No newline at end of file + ) + + \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index b5f2ce637..49d7d67f3 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -3,7 +3,44 @@ from typing import Dict, List, Optional, Type from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode -class TextSplittersFrontNode(FrontendNode): +class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: - pass \ No newline at end of file + pass + + """ + signature["template"]["documents"] = { + "type": "BaseLoader", + "required": True, + "show": True, + "name": "documents", + } + + signature["template"]["separator"] = { + "type": "str", + "required": True, + "show": True, + "value": ".", + "name": "separator", + "display_name": "Separator", + } + + signature["template"]["chunk_size"] = { + "type": "int", + "required": True, + "show": True, + "value": 4000, + "name": "chunk_size", + "display_name": "Chunk Size", + } + + signature["template"]["chunk_overlap"] = { + "type": "int", + "required": True, + "show": True, + "value": 200, + "name": "chunk_overlap", + "display_name": "Chunk Overlap", + } + """ + \ No newline at end of file From cd2944bac60f1d71d4f6b4f969de19e9c40e42cc Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Wed, 31 May 2023 15:40:09 -0300 Subject: [PATCH 03/69] added add_extra_fields method to textsplitters --- src/backend/langflow/interface/chains/base.py | 1 - .../interface/document_loaders/base.py | 10 +- .../langflow/interface/text_splitters/base.py | 6 +- .../template/frontend_node/documentloaders.py | 114 ++++++------------ .../template/frontend_node/textsplitters.py | 82 ++++++------- 5 files changed, 83 insertions(+), 130 deletions(-) diff --git a/src/backend/langflow/interface/chains/base.py b/src/backend/langflow/interface/chains/base.py index ad66f07b2..1b95d4118 100644 --- a/src/backend/langflow/interface/chains/base.py +++ b/src/backend/langflow/interface/chains/base.py @@ -10,7 +10,6 @@ from langflow.utils.util import build_template_from_class # Assuming necessary imports for Field, Template, and FrontendNode classes - class ChainCreator(LangChainTypeCreator): type_name: str = "chains" diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index 0a0efbb71..5219fbd13 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -1,30 +1,28 @@ from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator -from langflow.template.field.base import TemplateField from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode from langflow.interface.custom_lists import documentloaders_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class + class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" @property - def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]: + def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]: return DocumentLoaderFrontNode @property def type_to_loader_dict(self) -> Dict: return documentloaders_type_to_cls_dict - + def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a document loader.""" try: - return build_template_from_class( - name, documentloaders_type_to_cls_dict - ) + return build_template_from_class(name, documentloaders_type_to_cls_dict) except ValueError as exc: raise ValueError(f"Documment Loader {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/interface/text_splitters/base.py b/src/backend/langflow/interface/text_splitters/base.py index 11dfaca88..203f30086 100644 --- a/src/backend/langflow/interface/text_splitters/base.py +++ b/src/backend/langflow/interface/text_splitters/base.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator -from langflow.template.frontend_node.textsplitters import VectorStoreFrontendNode +from langflow.template.frontend_node.textsplitters import TextSplittersFrontendNode from langflow.interface.custom_lists import textsplitter_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger @@ -12,8 +12,8 @@ class TextSplitterCreator(LangChainTypeCreator): type_name: str = "textsplitters" @property - def frontend_node_class(self) -> Type[VectorStoreFrontendNode]: - return VectorStoreFrontendNode + def frontend_node_class(self) -> Type[TextSplittersFrontendNode]: + return TextSplittersFrontendNode @property def type_to_loader_dict(self) -> Dict: diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index 8d924f0fe..48c07a188 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -1,49 +1,32 @@ -from typing import Dict, List, Optional, Type - from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode -class DocumentLoaderFrontNode(FrontendNode): +class DocumentLoaderFrontNode(FrontendNode): @staticmethod - def build_template(suffixes: list, fileTypes: list, name: str = "file_path" - ) -> TemplateField: - """Build a template field for a document loader.""" - return TemplateField( - field_type="file", - required=True, - show=True, - name=name, - value="", - suffixes=suffixes, - fileTypes=fileTypes, - ) - + def build_template( + suffixes: list, fileTypes: list, name: str = "file_path" + ) -> TemplateField: + """Build a template field for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) + file_path_templates = { - "AirbyteJSONLoader": build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": build_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": build_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": build_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": build_template( - suffixes=[".html"], fileTypes=["html"] - ), + "AirbyteJSONLoader": build_template(suffixes=[".json"], fileTypes=["json"]), + "CoNLLULoader": build_template(suffixes=[".csv"], fileTypes=["csv"]), + "CSVLoader": build_template(suffixes=[".csv"], fileTypes=["csv"]), + "UnstructuredEmailLoader": build_template(suffixes=[".eml"], fileTypes=["eml"]), + "EverNoteLoader": build_template(suffixes=[".xml"], fileTypes=["xml"]), + "FacebookChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]), + "GutenbergLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]), + "BSHTMLLoader": build_template(suffixes=[".html"], fileTypes=["html"]), "UnstructuredHTMLLoader": build_template( suffixes=[".html"], fileTypes=["html"] ), @@ -54,28 +37,21 @@ class DocumentLoaderFrontNode(FrontendNode): "UnstructuredMarkdownLoader": build_template( suffixes=[".md"], fileTypes=["md"] ), - "PyPDFLoader": build_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), + "PyPDFLoader": build_template(suffixes=[".pdf"], fileTypes=["pdf"]), "UnstructuredPowerPointLoader": build_template( suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] ), - "SRTLoader": build_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": build_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": build_template( - suffixes=[".txt"], fileTypes=["txt"] - ), + "SRTLoader": build_template(suffixes=[".srt"], fileTypes=["srt"]), + "TelegramChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]), + "TextLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]), "UnstructuredWordDocumentLoader": build_template( suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] ), } - - def add_extra_fields(self) -> None: - if self.template.type_name in self.file_path_templates: + + def add_extra_fields(self) -> None: + name = None + if self.template.type_name in self.file_path_templates: self.template.add_field(self.file_path_templates[self.template.type_name]) elif self.template.type_name in { "WebBaseLoader", @@ -85,37 +61,19 @@ class DocumentLoaderFrontNode(FrontendNode): "IFixitLoader", "IMSDbLoader", }: - self.template.add_field( - TemplateField( - field_type="str", - required=True, - show=True, - name="web_path", - value="", - display_name="Web Page", - ) - ) + name = "web_path" elif self.template.type_name in {"GitbookLoader"}: - self.template.add_field( - TemplateField( - field_type="str", - required=True, - show=True, - name="web_page", - value="", - display_name="Web Page", - ) - ) + name = "web_page" elif self.template.type_name in {"ReadTheDocsLoader"}: + name = "path" + if name: self.template.add_field( TemplateField( field_type="str", required=True, show=True, - name="path", + name=name, value="", display_name="Web Page", ) ) - - \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index 49d7d67f3..f0bed10b0 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -1,46 +1,44 @@ -from typing import Dict, List, Optional, Type - from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode -class VectorStoreFrontendNode(FrontendNode): - + +class TextSplittersFrontendNode(FrontendNode): def add_extra_fields(self) -> None: - pass - - """ - signature["template"]["documents"] = { - "type": "BaseLoader", - "required": True, - "show": True, - "name": "documents", - } - - signature["template"]["separator"] = { - "type": "str", - "required": True, - "show": True, - "value": ".", - "name": "separator", - "display_name": "Separator", - } - - signature["template"]["chunk_size"] = { - "type": "int", - "required": True, - "show": True, - "value": 4000, - "name": "chunk_size", - "display_name": "Chunk Size", - } - - signature["template"]["chunk_overlap"] = { - "type": "int", - "required": True, - "show": True, - "value": 200, - "name": "chunk_overlap", - "display_name": "Chunk Overlap", - } - """ - \ No newline at end of file + self.template.add_field( + TemplateField( + field_type="BaseLoader", + required=True, + show=True, + name="documents", + ) + ) + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + value=".", + name="separator", + display_name="Separator", + ) + ) + self.template.add_field( + TemplateField( + field_type="int", + required=True, + show=True, + value=1000, + name="chunk_size", + display_name="Chunk Size", + ) + ) + self.template.add_field( + TemplateField( + field_type="int", + required=True, + show=True, + value=200, + name="chunk_overlap", + display_name="Chunk Overlap", + ) + ) From a662c5a6ccda92ff08a80467a3c39826372361b6 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Thu, 1 Jun 2023 16:21:11 -0300 Subject: [PATCH 04/69] reformatted with black --- src/backend/langflow/interface/chains/base.py | 1 + src/backend/langflow/template/frontend_node/vectorstores.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/backend/langflow/interface/chains/base.py b/src/backend/langflow/interface/chains/base.py index 1b95d4118..ad66f07b2 100644 --- a/src/backend/langflow/interface/chains/base.py +++ b/src/backend/langflow/interface/chains/base.py @@ -10,6 +10,7 @@ from langflow.utils.util import build_template_from_class # Assuming necessary imports for Field, Template, and FrontendNode classes + class ChainCreator(LangChainTypeCreator): type_name: str = "chains" diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index e071c3d4c..1aefaf10c 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -3,6 +3,7 @@ from typing import Optional from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode + class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: if self.template.type_name == "Weaviate": From feb3423101f9895318ce01456604f34ba5daccfd Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Wed, 7 Jun 2023 16:38:23 -0300 Subject: [PATCH 05/69] Added RecursiveCharacterTextSplitter method --- src/backend/langflow/config.yaml | 2 +- .../langflow/template/frontend_node/textsplitters.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 4060d1f3e..5e6a5cbed 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -65,7 +65,7 @@ prompts: - ZeroShotPrompt textsplitters: - CharacterTextSplitter - # - RecursiveCharacterTextSplitter + - RecursiveCharacterTextSplitter # - LatexTextSplitter # - PythonCodeTextSplitter toolkits: diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index f0bed10b0..03880379d 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -12,13 +12,18 @@ class TextSplittersFrontendNode(FrontendNode): name="documents", ) ) + name = "separator" + if self.template.type_name == "CharacterTextSplitter": + name = "separator" + elif self.template.type_name == "RecursiveCharacterTextSplitter": + name = "separators" self.template.add_field( TemplateField( field_type="str", required=True, show=True, value=".", - name="separator", + name=name, display_name="Separator", ) ) From 1c98506a5f334a97f941d06acfadbb23f76b2838 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Wed, 7 Jun 2023 16:40:46 -0300 Subject: [PATCH 06/69] Added RecursiveCharacterTextSplitter method --- src/backend/langflow/config.yaml | 2 +- src/backend/langflow/template/frontend_node/textsplitters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 5e6a5cbed..e03b2b1be 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -52,7 +52,7 @@ llms: - OpenAI # - AzureOpenAI - ChatOpenAI - - LlamaCpp + - LlamaCpp - CTransformers - Cohere memories: diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index 03880379d..b784618c0 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -10,7 +10,7 @@ class TextSplittersFrontendNode(FrontendNode): required=True, show=True, name="documents", - ) + ) ) name = "separator" if self.template.type_name == "CharacterTextSplitter": From 259b66bb004df0f0e87955ea57709bbfd8bd4767 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 16:41:48 -0300 Subject: [PATCH 07/69] fix: vertex_type now comes directly from template.type_name --- src/backend/langflow/graph/vertex/base.py | 7 ++++--- src/backend/langflow/interface/loading.py | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/backend/langflow/graph/vertex/base.py b/src/backend/langflow/graph/vertex/base.py index 04dadab85..0ea2e0779 100644 --- a/src/backend/langflow/graph/vertex/base.py +++ b/src/backend/langflow/graph/vertex/base.py @@ -48,9 +48,10 @@ class Vertex: ] template_dict = self.data["node"]["template"] - self.vertex_type = ( - self.data["type"] if "Tool" not in self.output else template_dict["_type"] - ) + # self.vertex_type = ( + # self.data["type"] if "Tool" not in self.output else template_dict["_type"] + # ) + self.vertex_type = template_dict["_type"] if self.base_type is None: for base_type, value in ALL_TYPES_DICT.items(): if self.vertex_type in value: diff --git a/src/backend/langflow/interface/loading.py b/src/backend/langflow/interface/loading.py index d0ec4b845..f8d26ffca 100644 --- a/src/backend/langflow/interface/loading.py +++ b/src/backend/langflow/interface/loading.py @@ -32,8 +32,7 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any: params = convert_params_to_sets(params) params = convert_kwargs(params) if node_type in CUSTOM_AGENTS: - custom_agent = CUSTOM_AGENTS.get(node_type) - if custom_agent: + if custom_agent := CUSTOM_AGENTS.get(node_type): return custom_agent.initialize(**params) class_object = import_by_type(_type=base_type, name=node_type) From af6a8b902a0fe3ae7c0a2074672de5006e75f5da Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 16:42:12 -0300 Subject: [PATCH 08/69] fix: initialize_agent now is called AgentInitializer --- src/backend/langflow/template/frontend_node/agents.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/template/frontend_node/agents.py b/src/backend/langflow/template/frontend_node/agents.py index 451dd7eca..101fac7ff 100644 --- a/src/backend/langflow/template/frontend_node/agents.py +++ b/src/backend/langflow/template/frontend_node/agents.py @@ -154,9 +154,9 @@ class CSVAgentNode(FrontendNode): class InitializeAgentNode(FrontendNode): - name: str = "initialize_agent" + name: str = "AgentInitializer" template: Template = Template( - type_name="initailize_agent", + type_name="initialize_agent", fields=[ TemplateField( field_type="str", From 2ca750f6541a7c8392cd86ccb9f520363b04e8b0 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 19:45:15 -0300 Subject: [PATCH 09/69] feat: add BaseRetriever as base class for VectorStores --- src/backend/langflow/template/frontend_node/vectorstores.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index 1aefaf10c..d04936a8b 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -20,6 +20,9 @@ class VectorStoreFrontendNode(FrontendNode): self.template.add_field(extra_field) + def add_extra_base_classes(self) -> None: + self.base_classes.append("BaseRetriever") + @staticmethod def format_field(field: TemplateField, name: Optional[str] = None) -> None: FrontendNode.format_field(field, name) From b5ab95c4cce84af19b99bd26e0b4907fdb2ddaa4 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 19:46:51 -0300 Subject: [PATCH 10/69] =?UTF-8?q?=F0=9F=94=A8=20refactor(custom.py):=20rem?= =?UTF-8?q?ove=20unused=20code=20=E2=9C=A8=20feat(frontend=5Fnode):=20add?= =?UTF-8?q?=20extra=20fields=20to=20MemoryFrontendNode=20and=20ChainFronte?= =?UTF-8?q?ndNode=20The=20unused=20code=20in=20custom.py=20has=20been=20re?= =?UTF-8?q?moved.=20The=20MemoryFrontendNode=20and=20ChainFrontendNode=20c?= =?UTF-8?q?lasses=20have=20been=20updated=20to=20include=20additional=20fi?= =?UTF-8?q?elds=20that=20are=20required=20for=20their=20respective=20templ?= =?UTF-8?q?ates.=20The=20MemoryFrontendNode=20now=20has=20a=20return=5Fmes?= =?UTF-8?q?sages=20field,=20and=20the=20ChainFrontendNode=20now=20has=20a?= =?UTF-8?q?=20memory=20field.=20These=20fields=20are=20optional=20and=20ca?= =?UTF-8?q?n=20be=20toggled=20on=20or=20off=20as=20required.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/prompts/custom.py | 4 -- .../langflow/template/frontend_node/base.py | 3 ++ .../langflow/template/frontend_node/chains.py | 52 +++++++++++++++++++ .../template/frontend_node/memories.py | 18 +++++++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/backend/langflow/interface/prompts/custom.py b/src/backend/langflow/interface/prompts/custom.py index 286210271..ef16f1474 100644 --- a/src/backend/langflow/interface/prompts/custom.py +++ b/src/backend/langflow/interface/prompts/custom.py @@ -71,7 +71,3 @@ Human: {input} CUSTOM_PROMPTS: Dict[str, Type[BaseCustomPrompt]] = { "SeriesCharacterPrompt": SeriesCharacterPrompt } - -if __name__ == "__main__": - prompt = SeriesCharacterPrompt(character="Harry Potter", series="Harry Potter") - print(prompt.template) diff --git a/src/backend/langflow/template/frontend_node/base.py b/src/backend/langflow/template/frontend_node/base.py index 6d00cead0..074a85499 100644 --- a/src/backend/langflow/template/frontend_node/base.py +++ b/src/backend/langflow/template/frontend_node/base.py @@ -27,6 +27,9 @@ class FrontendNode(BaseModel): def add_extra_fields(self) -> None: pass + def add_extra_base_classes(self) -> None: + pass + @staticmethod def format_field(field: TemplateField, name: Optional[str] = None) -> None: """Formats a given field based on its attributes and value.""" diff --git a/src/backend/langflow/template/frontend_node/chains.py b/src/backend/langflow/template/frontend_node/chains.py index cfcda2ef4..cb06c90f0 100644 --- a/src/backend/langflow/template/frontend_node/chains.py +++ b/src/backend/langflow/template/frontend_node/chains.py @@ -2,10 +2,24 @@ from typing import Optional from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode +from langflow.template.frontend_node.constants import QA_CHAIN_TYPES from langflow.template.template.base import Template class ChainFrontendNode(FrontendNode): + def add_extra_fields(self) -> None: + if self.template.type_name == "ConversationalRetrievalChain": + # add memory + self.template.add_field( + TemplateField( + field_type="BaseChatMemory", + required=False, + show=True, + name="memory", + advanced=False, + ) + ) + @staticmethod def format_field(field: TemplateField, name: Optional[str] = None) -> None: FrontendNode.format_field(field, name) @@ -155,3 +169,41 @@ class MidJourneyPromptChainNode(FrontendNode): "ConversationChain", "MidJourneyPromptChain", ] + + +class CombineDocsChainNode(FrontendNode): + name: str = "CombineDocsChain" + template: Template = Template( + type_name="load_qa_chain", + fields=[ + TemplateField( + field_type="str", + required=True, + is_list=True, + show=True, + multiline=False, + options=QA_CHAIN_TYPES, + value=QA_CHAIN_TYPES[0], + name="chain_type", + advanced=False, + ), + TemplateField( + field_type="BaseLanguageModel", + required=True, + show=True, + name="llm", + display_name="LLM", + advanced=False, + ), + ], + ) + description: str = """Construct a zero shot agent from an LLM and tools.""" + base_classes: list[str] = ["BaseCombineDocumentsChain", "function"] + + def to_dict(self): + return super().to_dict() + + @staticmethod + def format_field(field: TemplateField, name: Optional[str] = None) -> None: + # do nothing and don't return anything + pass diff --git a/src/backend/langflow/template/frontend_node/memories.py b/src/backend/langflow/template/frontend_node/memories.py index 91d892627..20c3c9272 100644 --- a/src/backend/langflow/template/frontend_node/memories.py +++ b/src/backend/langflow/template/frontend_node/memories.py @@ -5,6 +5,20 @@ from langflow.template.frontend_node.base import FrontendNode class MemoryFrontendNode(FrontendNode): + #! Needs testing + def add_extra_fields(self) -> None: + # add return_messages field + self.template.add_field( + TemplateField( + field_type="bool", + required=False, + show=True, + name="return_messages", + advanced=False, + value=False, + ) + ) + @staticmethod def format_field(field: TemplateField, name: Optional[str] = None) -> None: FrontendNode.format_field(field, name) @@ -18,3 +32,7 @@ class MemoryFrontendNode(FrontendNode): field.value = 10 field.display_name = "Memory Size" field.password = False + if field.name == "return_messages": + field.required = False + field.show = True + field.advanced = False From c8125b338675c74db132f4fc4fffae5850c18296 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 19:48:16 -0300 Subject: [PATCH 11/69] =?UTF-8?q?=F0=9F=94=80=20refactor(customs.py):=20ad?= =?UTF-8?q?d=20load=5Fqa=5Fchain=20to=20CUSTOM=5FNODES=20=F0=9F=94=80=20re?= =?UTF-8?q?factor(custom=5Flists.py):=20merge=20CUSTOM=5FAGENTS=20and=20CU?= =?UTF-8?q?STOM=5FCHAINS=20into=20CUSTOM=5FNODES=20=F0=9F=94=80=20refactor?= =?UTF-8?q?(loading.py):=20add=20instantiate=5Fchains=20function=20to=20in?= =?UTF-8?q?stantiate=20chains=20=F0=9F=94=80=20refactor(base.py):=20add=20?= =?UTF-8?q?CustomChain=20class=20The=20changes=20add=20support=20for=20a?= =?UTF-8?q?=20new=20chain=20called=20load=5Fqa=5Fchain=20to=20CUSTOM=5FNOD?= =?UTF-8?q?ES.=20CUSTOM=5FAGENTS=20and=20CUSTOM=5FCHAINS=20are=20merged=20?= =?UTF-8?q?into=20CUSTOM=5FNODES.=20A=20new=20function=20called=20instanti?= =?UTF-8?q?ate=5Fchains=20is=20added=20to=20instantiate=20chains.=20A=20ne?= =?UTF-8?q?w=20class=20called=20CustomChain=20is=20added=20to=20the=20base?= =?UTF-8?q?.py=20file.=20This=20class=20is=20used=20to=20define=20a=20cust?= =?UTF-8?q?om=20chain.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/custom/customs.py | 5 +++- src/backend/langflow/interface/base.py | 20 +++++++++++++ .../langflow/interface/custom_lists.py | 11 ++++--- src/backend/langflow/interface/loading.py | 29 +++++++++++++++++-- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/backend/langflow/custom/customs.py b/src/backend/langflow/custom/customs.py index 92e1fc2d8..e315d27fd 100644 --- a/src/backend/langflow/custom/customs.py +++ b/src/backend/langflow/custom/customs.py @@ -2,7 +2,9 @@ from langflow.template import frontend_node # These should always be instantiated CUSTOM_NODES = { - "prompts": {"ZeroShotPrompt": frontend_node.prompts.ZeroShotPromptNode()}, + "prompts": { + "ZeroShotPrompt": frontend_node.prompts.ZeroShotPromptNode(), + }, "tools": { "PythonFunctionTool": frontend_node.tools.PythonFunctionToolNode(), "PythonFunction": frontend_node.tools.PythonFunctionNode(), @@ -23,6 +25,7 @@ CUSTOM_NODES = { "SeriesCharacterChain": frontend_node.chains.SeriesCharacterChainNode(), "TimeTravelGuideChain": frontend_node.chains.TimeTravelGuideChainNode(), "MidJourneyPromptChain": frontend_node.chains.MidJourneyPromptChainNode(), + "load_qa_chain": frontend_node.chains.CombineDocsChainNode(), }, } diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index 08cbc6681..df03950af 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Type, Union +from langchain.chains.base import Chain from pydantic import BaseModel @@ -81,5 +82,24 @@ class LangChainTypeCreator(BaseModel, ABC): ) signature.add_extra_fields() + signature.add_extra_base_classes() return signature + + +class CustomChain(Chain, ABC): + """Custom chain""" + + @staticmethod + def function_name(): + return "CustomChain" + + @classmethod + def initialize(cls, *args, **kwargs): + pass + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def run(self, *args, **kwargs): + return super().run(*args, **kwargs) diff --git a/src/backend/langflow/interface/custom_lists.py b/src/backend/langflow/interface/custom_lists.py index 34bc0103e..8bde1565c 100644 --- a/src/backend/langflow/interface/custom_lists.py +++ b/src/backend/langflow/interface/custom_lists.py @@ -2,7 +2,6 @@ import inspect from typing import Any from langchain import ( - chains, document_loaders, embeddings, llms, @@ -15,6 +14,8 @@ from langchain.chat_models import AzureChatOpenAI, ChatOpenAI from langchain.chat_models import ChatAnthropic from langflow.interface.importing.utils import import_class +from langflow.interface.agents.custom import CUSTOM_AGENTS +from langflow.interface.chains.custom import CUSTOM_CHAINS ## LLMs llm_type_to_cls_dict = llms.type_to_cls_dict @@ -22,11 +23,6 @@ llm_type_to_cls_dict["anthropic-chat"] = ChatAnthropic # type: ignore llm_type_to_cls_dict["azure-chat"] = AzureChatOpenAI # type: ignore llm_type_to_cls_dict["openai-chat"] = ChatOpenAI # type: ignore -## Chains -chain_type_to_cls_dict: dict[str, Any] = { - chain_name: import_class(f"langchain.chains.{chain_name}") - for chain_name in chains.__all__ -} ## Toolkits toolkit_type_to_loader_dict: dict[str, Any] = { @@ -73,3 +69,6 @@ documentloaders_type_to_cls_dict: dict[str, Any] = { textsplitter_type_to_cls_dict: dict[str, Any] = dict( inspect.getmembers(text_splitter, inspect.isclass) ) + +# merge CUSTOM_AGENTS and CUSTOM_CHAINS +CUSTOM_NODES = {**CUSTOM_AGENTS, **CUSTOM_CHAINS} diff --git a/src/backend/langflow/interface/loading.py b/src/backend/langflow/interface/loading.py index f8d26ffca..80f451f03 100644 --- a/src/backend/langflow/interface/loading.py +++ b/src/backend/langflow/interface/loading.py @@ -19,9 +19,10 @@ from langchain.chains.loading import load_chain_from_config from langchain.llms.loading import load_llm_from_config from pydantic import ValidationError -from langflow.interface.agents.custom import CUSTOM_AGENTS +from langflow.interface.custom_lists import CUSTOM_NODES from langflow.interface.importing.utils import get_function, import_by_type from langflow.interface.toolkits.base import toolkits_creator +from langflow.interface.chains.base import chain_creator from langflow.interface.types import get_type_list from langflow.interface.utils import load_file_into_dict from langflow.utils import util, validate @@ -31,8 +32,8 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any: """Instantiate class from module type and key, and params""" params = convert_params_to_sets(params) params = convert_kwargs(params) - if node_type in CUSTOM_AGENTS: - if custom_agent := CUSTOM_AGENTS.get(node_type): + if node_type in CUSTOM_NODES: + if custom_agent := CUSTOM_NODES.get(node_type): return custom_agent.initialize(**params) class_object = import_by_type(_type=base_type, name=node_type) @@ -77,10 +78,24 @@ def instantiate_based_on_type(class_object, base_type, node_type, params): return instantiate_textsplitter(class_object, params) elif base_type == "utilities": return instantiate_utility(node_type, class_object, params) + elif base_type == "chains": + return instantiate_chains(node_type, class_object, params) else: return class_object(**params) +def instantiate_chains(node_type, class_object, params): + if "retriever" in params and hasattr(params["retriever"], "as_retriever"): + params["retriever"] = params["retriever"].as_retriever() + if node_type in chain_creator.from_method_nodes: + method = chain_creator.from_method_nodes[node_type] + if class_method := getattr(class_object, method, None): + return class_method(**params) + raise ValueError(f"Method {method} not found in {class_object}") + + return class_object(**params) + + def instantiate_agent(class_object, params): return load_agent_executor(class_object, params) @@ -141,6 +156,14 @@ def instantiate_vectorstore(class_object, params): "The source you provided did not load correctly or was empty." "This may cause an error in the vectorstore." ) + # Chroma requires all metadata values to not be None + if class_object.__name__ == "Chroma": + for doc in params["documents"]: + if doc.metadata is None: + doc.metadata = {} + for key, value in doc.metadata.items(): + if value is None: + doc.metadata[key] = "" return class_object.from_documents(**params) From f0975ddf63fa54e8d9f46ecdc8d4b640afda0e75 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 19:48:51 -0300 Subject: [PATCH 12/69] =?UTF-8?q?=F0=9F=9A=80=20feat(langflow):=20add=20ne?= =?UTF-8?q?w=20chains=20to=20config.yaml=20and=20custom=20chains=20to=20in?= =?UTF-8?q?terface/chains/custom.py=20=E2=9C=A8=20feat(langflow):=20add=20?= =?UTF-8?q?new=20chains=20to=20config.yaml=20and=20custom=20chains=20to=20?= =?UTF-8?q?interface/chains/custom.py=20The=20following=20chains=20were=20?= =?UTF-8?q?added=20to=20the=20config.yaml=20file:=20RetrievalQA,=20Retriev?= =?UTF-8?q?alQAWithSourcesChain,=20QAWithSourcesChain,=20ConversationalRet?= =?UTF-8?q?rievalChain,=20and=20CombineDocsChain.=20These=20chains=20were?= =?UTF-8?q?=20added=20to=20improve=20the=20functionality=20of=20the=20appl?= =?UTF-8?q?ication=20and=20provide=20more=20options=20for=20users.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In addition, custom chains were added to the interface/chains/custom.py file. The CombineDocsChain was added to allow users to combine multiple documents into a single document for use in the question answering chains. The QA_CHAIN_TYPES constant was also added to the frontend_node/constants.py file to provide a list of available question answering chain types. --- src/backend/langflow/config.yaml | 5 +++ .../langflow/interface/agents/custom.py | 32 +++----------- src/backend/langflow/interface/chains/base.py | 43 +++++++++++++++---- .../langflow/interface/chains/custom.py | 35 ++++++++++++--- .../template/frontend_node/constants.py | 2 + 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 5e02e2984..e253ed8e4 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -16,6 +16,11 @@ chains: - MidJourneyPromptChain - TimeTravelGuideChain - SQLDatabaseChain + - RetrievalQA + - RetrievalQAWithSourcesChain + - QAWithSourcesChain + - ConversationalRetrievalChain + - CombineDocsChain documentloaders: - AirbyteJSONLoader - CoNLLULoader diff --git a/src/backend/langflow/interface/agents/custom.py b/src/backend/langflow/interface/agents/custom.py index 3aaa132d4..a0a5c243f 100644 --- a/src/backend/langflow/interface/agents/custom.py +++ b/src/backend/langflow/interface/agents/custom.py @@ -1,4 +1,3 @@ -from abc import ABC from typing import Any, List, Optional from langchain import LLMChain @@ -33,27 +32,10 @@ from langchain.memory.chat_memory import BaseChatMemory from langchain.sql_database import SQLDatabase from langchain.tools.python.tool import PythonAstREPLTool from langchain.tools.sql_database.prompt import QUERY_CHECKER +from langflow.interface.base import CustomChain -class CustomAgentExecutor(AgentExecutor, ABC): - """Custom agent executor""" - - @staticmethod - def function_name(): - return "CustomAgentExecutor" - - @classmethod - def initialize(cls, *args, **kwargs): - pass - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def run(self, *args, **kwargs): - return super().run(*args, **kwargs) - - -class JsonAgent(CustomAgentExecutor): +class JsonAgent(CustomChain): """Json agent""" @staticmethod @@ -89,7 +71,7 @@ class JsonAgent(CustomAgentExecutor): return super().run(*args, **kwargs) -class CSVAgent(CustomAgentExecutor): +class CSVAgent(CustomChain): """CSV agent""" @staticmethod @@ -137,7 +119,7 @@ class CSVAgent(CustomAgentExecutor): return super().run(*args, **kwargs) -class VectorStoreAgent(CustomAgentExecutor): +class VectorStoreAgent(CustomChain): """Vector Store agent""" @staticmethod @@ -175,7 +157,7 @@ class VectorStoreAgent(CustomAgentExecutor): return super().run(*args, **kwargs) -class SQLAgent(CustomAgentExecutor): +class SQLAgent(CustomChain): """SQL agent""" @staticmethod @@ -247,7 +229,7 @@ class SQLAgent(CustomAgentExecutor): return super().run(*args, **kwargs) -class VectorStoreRouterAgent(CustomAgentExecutor): +class VectorStoreRouterAgent(CustomChain): """Vector Store Router Agent""" @staticmethod @@ -286,7 +268,7 @@ class VectorStoreRouterAgent(CustomAgentExecutor): return super().run(*args, **kwargs) -class InitializeAgent(CustomAgentExecutor): +class InitializeAgent(CustomChain): """Implementation of initialize_agent function""" @staticmethod diff --git a/src/backend/langflow/interface/chains/base.py b/src/backend/langflow/interface/chains/base.py index ad66f07b2..054ca2775 100644 --- a/src/backend/langflow/interface/chains/base.py +++ b/src/backend/langflow/interface/chains/base.py @@ -1,12 +1,13 @@ -from typing import Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type from langflow.custom.customs import get_custom_nodes from langflow.interface.base import LangChainTypeCreator -from langflow.interface.custom_lists import chain_type_to_cls_dict +from langflow.interface.importing.utils import import_class from langflow.settings import settings from langflow.template.frontend_node.chains import ChainFrontendNode from langflow.utils.logger import logger -from langflow.utils.util import build_template_from_class +from langflow.utils.util import build_template_from_class, build_template_from_method +from langchain import chains # Assuming necessary imports for Field, Template, and FrontendNode classes @@ -18,10 +19,16 @@ class ChainCreator(LangChainTypeCreator): def frontend_node_class(self) -> Type[ChainFrontendNode]: return ChainFrontendNode + #! We need to find a better solution for this + from_method_nodes = {"ConversationalRetrievalChain": "from_llm"} + @property def type_to_loader_dict(self) -> Dict: if self.type_dict is None: - self.type_dict = chain_type_to_cls_dict + self.type_dict: dict[str, Any] = { + chain_name: import_class(f"langchain.chains.{chain_name}") + for chain_name in chains.__all__ + } from langflow.interface.chains.custom import CUSTOM_CHAINS self.type_dict.update(CUSTOM_CHAINS) @@ -37,20 +44,38 @@ class ChainCreator(LangChainTypeCreator): try: if name in get_custom_nodes(self.type_name).keys(): return get_custom_nodes(self.type_name)[name] + elif name in self.from_method_nodes.keys(): + return build_template_from_method( + name, + type_to_cls_dict=self.type_to_loader_dict, + method_name=self.from_method_nodes[name], + add_function=True, + ) return build_template_from_class( name, self.type_to_loader_dict, add_function=True ) except ValueError as exc: - raise ValueError("Chain not found") from exc + raise ValueError(f"Chain {name} not found: {exc}") from exc except AttributeError as exc: logger.error(f"Chain {name} not loaded: {exc}") return None - def to_list(self) -> List[str]: - custom_chains = list(get_custom_nodes("chains").keys()) - default_chains = list(self.type_to_loader_dict.keys()) + # def to_list(self) -> List[str]: + # custom_chains = list(get_custom_nodes("chains").keys()) + # default_chains = list(self.type_to_loader_dict.keys()) - return default_chains + custom_chains + # return default_chains + custom_chains + + def to_list(self) -> List[str]: + names = [] + for _, chain in self.type_to_loader_dict.items(): + chain_name = ( + chain.function_name() + if hasattr(chain, "function_name") + else chain.__name__ + ) + names.append(chain_name) + return names chain_creator = ChainCreator() diff --git a/src/backend/langflow/interface/chains/custom.py b/src/backend/langflow/interface/chains/custom.py index ba4ba8b62..411042b0f 100644 --- a/src/backend/langflow/interface/chains/custom.py +++ b/src/backend/langflow/interface/chains/custom.py @@ -1,11 +1,13 @@ -from typing import Dict, Optional, Type +from typing import Dict, Optional, Type, Union from langchain.chains import ConversationChain from langchain.memory.buffer import ConversationBufferMemory from langchain.schema import BaseMemory +from langflow.interface.base import CustomChain from pydantic import Field, root_validator - +from langchain.chains.question_answering import load_qa_chain from langflow.interface.utils import extract_input_variables_from_prompt +from langchain.base_language import BaseLanguageModel DEFAULT_SUFFIX = """" Current conversation: @@ -14,7 +16,7 @@ Human: {input} {ai_prefix}""" -class BaseCustomChain(ConversationChain): +class BaseCustomConversationChain(ConversationChain): """BaseCustomChain is a chain you can use to have a conversation with a custom character.""" template: Optional[str] @@ -47,7 +49,7 @@ class BaseCustomChain(ConversationChain): return values -class SeriesCharacterChain(BaseCustomChain): +class SeriesCharacterChain(BaseCustomConversationChain): """SeriesCharacterChain is a chain you can use to have a conversation with a character from a series.""" character: str @@ -66,7 +68,7 @@ Human: {input} """Default memory store.""" -class MidJourneyPromptChain(BaseCustomChain): +class MidJourneyPromptChain(BaseCustomConversationChain): """MidJourneyPromptChain is a chain you can use to generate new MidJourney prompts.""" template: Optional[ @@ -84,7 +86,7 @@ class MidJourneyPromptChain(BaseCustomChain): AI:""" # noqa: E501 -class TimeTravelGuideChain(BaseCustomChain): +class TimeTravelGuideChain(BaseCustomConversationChain): template: Optional[ str ] = """I want you to act as my time travel guide. You are helpful and creative. I will provide you with the historical period or future time I want to visit and you will suggest the best events, sights, or people to experience. Provide the suggestions and any necessary information. @@ -94,7 +96,26 @@ class TimeTravelGuideChain(BaseCustomChain): AI:""" # noqa: E501 -CUSTOM_CHAINS: Dict[str, Type[ConversationChain]] = { +class CombineDocsChain(CustomChain): + """Implementation of initialize_agent function""" + + @staticmethod + def function_name(): + return "load_qa_chain" + + @classmethod + def initialize(cls, llm: BaseLanguageModel, chain_type: str): + return load_qa_chain(llm=llm, chain_type=chain_type) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def run(self, *args, **kwargs): + return super().run(*args, **kwargs) + + +CUSTOM_CHAINS: Dict[str, Type[Union[ConversationChain, CustomChain]]] = { + "CombineDocsChain": CombineDocsChain, "SeriesCharacterChain": SeriesCharacterChain, "MidJourneyPromptChain": MidJourneyPromptChain, "TimeTravelGuideChain": TimeTravelGuideChain, diff --git a/src/backend/langflow/template/frontend_node/constants.py b/src/backend/langflow/template/frontend_node/constants.py index ae08d3691..20b8a0c61 100644 --- a/src/backend/langflow/template/frontend_node/constants.py +++ b/src/backend/langflow/template/frontend_node/constants.py @@ -30,3 +30,5 @@ You are a good listener and you can talk about anything. """ HUMAN_PROMPT = "{input}" + +QA_CHAIN_TYPES = ["stuff", "map_reduce", "map_rerank", "refine"] From 04bd0f43fb0e621679b5bb42582455c09eedce56 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 21:14:40 -0300 Subject: [PATCH 13/69] =?UTF-8?q?=F0=9F=94=A8=20refactor(custom.py):=20ren?= =?UTF-8?q?ame=20CustomChain=20to=20CustomAgentExecutor=20=F0=9F=94=A8=20r?= =?UTF-8?q?efactor(base.py):=20add=20CustomAgentExecutor=20class=20and=20m?= =?UTF-8?q?ove=20CustomChain=20to=20base.py=20=F0=9F=94=A8=20refactor(cust?= =?UTF-8?q?om=5Flists.py):=20update=20reference=20to=20CustomChain=20to=20?= =?UTF-8?q?CustomAgentExecutor=20The=20CustomChain=20class=20has=20been=20?= =?UTF-8?q?renamed=20to=20CustomAgentExecutor=20to=20better=20reflect=20it?= =?UTF-8?q?s=20purpose.=20The=20class=20has=20been=20moved=20to=20base.py?= =?UTF-8?q?=20and=20a=20new=20CustomAgentExecutor=20class=20has=20been=20a?= =?UTF-8?q?dded=20to=20custom.py.=20The=20reference=20to=20CustomChain=20i?= =?UTF-8?q?n=20custom=5Flists.py=20has=20been=20updated=20to=20CustomAgent?= =?UTF-8?q?Executor.=20These=20changes=20improve=20the=20semantics=20of=20?= =?UTF-8?q?the=20code=20and=20make=20it=20easier=20to=20understand=20the?= =?UTF-8?q?=20purpose=20of=20the=20classes.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/agents/custom.py | 14 ++++++------- src/backend/langflow/interface/base.py | 20 ++++++++++++++++++- .../langflow/interface/custom_lists.py | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/backend/langflow/interface/agents/custom.py b/src/backend/langflow/interface/agents/custom.py index a0a5c243f..f86028985 100644 --- a/src/backend/langflow/interface/agents/custom.py +++ b/src/backend/langflow/interface/agents/custom.py @@ -32,10 +32,10 @@ from langchain.memory.chat_memory import BaseChatMemory from langchain.sql_database import SQLDatabase from langchain.tools.python.tool import PythonAstREPLTool from langchain.tools.sql_database.prompt import QUERY_CHECKER -from langflow.interface.base import CustomChain +from langflow.interface.base import CustomAgentExecutor -class JsonAgent(CustomChain): +class JsonAgent(CustomAgentExecutor): """Json agent""" @staticmethod @@ -71,7 +71,7 @@ class JsonAgent(CustomChain): return super().run(*args, **kwargs) -class CSVAgent(CustomChain): +class CSVAgent(CustomAgentExecutor): """CSV agent""" @staticmethod @@ -119,7 +119,7 @@ class CSVAgent(CustomChain): return super().run(*args, **kwargs) -class VectorStoreAgent(CustomChain): +class VectorStoreAgent(CustomAgentExecutor): """Vector Store agent""" @staticmethod @@ -157,7 +157,7 @@ class VectorStoreAgent(CustomChain): return super().run(*args, **kwargs) -class SQLAgent(CustomChain): +class SQLAgent(CustomAgentExecutor): """SQL agent""" @staticmethod @@ -229,7 +229,7 @@ class SQLAgent(CustomChain): return super().run(*args, **kwargs) -class VectorStoreRouterAgent(CustomChain): +class VectorStoreRouterAgent(CustomAgentExecutor): """Vector Store Router Agent""" @staticmethod @@ -268,7 +268,7 @@ class VectorStoreRouterAgent(CustomChain): return super().run(*args, **kwargs) -class InitializeAgent(CustomChain): +class InitializeAgent(CustomAgentExecutor): """Implementation of initialize_agent function""" @staticmethod diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index df03950af..3670bb8ae 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional, Type, Union from langchain.chains.base import Chain - +from langchain.agents import AgentExecutor from pydantic import BaseModel from langflow.template.field.base import TemplateField @@ -103,3 +103,21 @@ class CustomChain(Chain, ABC): def run(self, *args, **kwargs): return super().run(*args, **kwargs) + + +class CustomAgentExecutor(AgentExecutor, ABC): + """Custom chain""" + + @staticmethod + def function_name(): + return "CustomChain" + + @classmethod + def initialize(cls, *args, **kwargs): + pass + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def run(self, *args, **kwargs): + return super().run(*args, **kwargs) diff --git a/src/backend/langflow/interface/custom_lists.py b/src/backend/langflow/interface/custom_lists.py index 8bde1565c..fbdba0a9c 100644 --- a/src/backend/langflow/interface/custom_lists.py +++ b/src/backend/langflow/interface/custom_lists.py @@ -71,4 +71,4 @@ textsplitter_type_to_cls_dict: dict[str, Any] = dict( ) # merge CUSTOM_AGENTS and CUSTOM_CHAINS -CUSTOM_NODES = {**CUSTOM_AGENTS, **CUSTOM_CHAINS} +CUSTOM_NODES = {**CUSTOM_AGENTS, **CUSTOM_CHAINS} # type: ignore From 81d231c632755e19302f9a6bcc0d85c02640cdee Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 7 Jun 2023 21:36:40 -0300 Subject: [PATCH 14/69] =?UTF-8?q?=F0=9F=90=9B=20fix(vertex/base.py):=20fix?= =?UTF-8?q?=20vertex=5Ftype=20assignment=20to=20handle=20uppercase=20templ?= =?UTF-8?q?ate=20types=20=F0=9F=90=9B=20fix(interface/loading.py):=20fix?= =?UTF-8?q?=20custom=5Fnode=20instantiation=20to=20handle=20classes=20with?= =?UTF-8?q?out=20initialize=20method=20=E2=9C=A8=20feat(template/frontend?= =?UTF-8?q?=5Fnode/prompts.py):=20change=20type=5Fname=20to=20match=20clas?= =?UTF-8?q?s=20name=20=E2=9C=A8=20feat(template/frontend=5Fnode/tools.py):?= =?UTF-8?q?=20change=20type=5Fname=20to=20match=20class=20name=20?= =?UTF-8?q?=F0=9F=94=A5=20chore(test=5Fagents=5Ftemplate.py):=20remove=20t?= =?UTF-8?q?est=5Fagents=5Fsettings=20and=20update=20initialize=5Fagent=20t?= =?UTF-8?q?est=20The=20vertex=5Ftype=20assignment=20in=20the=20Vertex=20cl?= =?UTF-8?q?ass=20was=20not=20handling=20uppercase=20template=20types=20cor?= =?UTF-8?q?rectly.=20This=20has=20been=20fixed=20to=20handle=20both=20uppe?= =?UTF-8?q?rcase=20and=20lowercase=20types.=20The=20custom=5Fnode=20instan?= =?UTF-8?q?tiation=20in=20the=20instantiate=5Fclass=20function=20was=20not?= =?UTF-8?q?=20handling=20classes=20without=20an=20initialize=20method=20co?= =?UTF-8?q?rrectly.=20This=20has=20been=20fixed=20to=20instantiate=20the?= =?UTF-8?q?=20class=20directly=20if=20the=20initialize=20method=20is=20not?= =?UTF-8?q?=20present.=20The=20type=5Fname=20in=20the=20ZeroShotPromptNode?= =?UTF-8?q?=20and=20PythonFunctionToolNode=20classes=20have=20been=20chang?= =?UTF-8?q?ed=20to=20match=20the=20class=20name.=20The=20test=5Fagents=5Fs?= =?UTF-8?q?ettings=20test=20has=20been=20removed=20as=20it=20is=20no=20lon?= =?UTF-8?q?ger=20necessary=20and=20the=20initialize=5Fagent=20test=20has?= =?UTF-8?q?=20been=20updated=20to=20match=20the=20new=20AgentInitializer?= =?UTF-8?q?=20class=20name.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/graph/vertex/base.py | 10 ++++++---- src/backend/langflow/interface/loading.py | 6 ++++-- .../langflow/template/frontend_node/prompts.py | 2 +- .../langflow/template/frontend_node/tools.py | 2 +- tests/test_agents_template.py | 13 +------------ 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/backend/langflow/graph/vertex/base.py b/src/backend/langflow/graph/vertex/base.py index 0ea2e0779..31cc11968 100644 --- a/src/backend/langflow/graph/vertex/base.py +++ b/src/backend/langflow/graph/vertex/base.py @@ -48,10 +48,12 @@ class Vertex: ] template_dict = self.data["node"]["template"] - # self.vertex_type = ( - # self.data["type"] if "Tool" not in self.output else template_dict["_type"] - # ) - self.vertex_type = template_dict["_type"] + self.vertex_type = ( + self.data["type"] + if "Tool" not in self.output or template_dict["_type"].islower() + else template_dict["_type"] + ) + # self.vertex_type = template_dict["_type"] if self.base_type is None: for base_type, value in ALL_TYPES_DICT.items(): if self.vertex_type in value: diff --git a/src/backend/langflow/interface/loading.py b/src/backend/langflow/interface/loading.py index 80f451f03..a765d3b9b 100644 --- a/src/backend/langflow/interface/loading.py +++ b/src/backend/langflow/interface/loading.py @@ -33,8 +33,10 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any: params = convert_params_to_sets(params) params = convert_kwargs(params) if node_type in CUSTOM_NODES: - if custom_agent := CUSTOM_NODES.get(node_type): - return custom_agent.initialize(**params) + if custom_node := CUSTOM_NODES.get(node_type): + if hasattr(custom_node, "initialize"): + return custom_node.initialize(**params) + return custom_node(**params) class_object = import_by_type(_type=base_type, name=node_type) return instantiate_based_on_type(class_object, base_type, node_type, params) diff --git a/src/backend/langflow/template/frontend_node/prompts.py b/src/backend/langflow/template/frontend_node/prompts.py index 8738f1795..da5d2a300 100644 --- a/src/backend/langflow/template/frontend_node/prompts.py +++ b/src/backend/langflow/template/frontend_node/prompts.py @@ -74,7 +74,7 @@ class BasePromptFrontendNode(FrontendNode): class ZeroShotPromptNode(BasePromptFrontendNode): name: str = "ZeroShotPrompt" template: Template = Template( - type_name="zero_shot", + type_name="ZeroShotPrompt", fields=[ TemplateField( field_type="str", diff --git a/src/backend/langflow/template/frontend_node/tools.py b/src/backend/langflow/template/frontend_node/tools.py index 3094f3568..dd312f906 100644 --- a/src/backend/langflow/template/frontend_node/tools.py +++ b/src/backend/langflow/template/frontend_node/tools.py @@ -108,7 +108,7 @@ class PythonFunctionToolNode(FrontendNode): class PythonFunctionNode(FrontendNode): name: str = "PythonFunction" template: Template = Template( - type_name="python_function", + type_name="PythonFunction", fields=[ TemplateField( field_type="code", diff --git a/tests/test_agents_template.py b/tests/test_agents_template.py index 8e181711f..e58007238 100644 --- a/tests/test_agents_template.py +++ b/tests/test_agents_template.py @@ -1,15 +1,4 @@ from fastapi.testclient import TestClient -from langflow.settings import settings - - -# check that all agents are in settings.agents -# are in json_response["agents"] -def test_agents_settings(client: TestClient): - response = client.get("api/v1/all") - assert response.status_code == 200 - json_response = response.json() - agents = json_response["agents"] - assert set(agents.keys()) == set(settings.agents) def test_zero_shot_agent(client: TestClient): @@ -131,7 +120,7 @@ def test_initialize_agent(client: TestClient): json_response = response.json() agents = json_response["agents"] - initialize_agent = agents["initialize_agent"] + initialize_agent = agents["AgentInitializer"] assert initialize_agent["base_classes"] == ["AgentExecutor", "function"] template = initialize_agent["template"] From 85d2f9cce8bda98a747898e50adc0f726dd54fc7 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Thu, 8 Jun 2023 00:14:33 -0300 Subject: [PATCH 15/69] =?UTF-8?q?=F0=9F=94=A8=20refactor(base.py):=20make?= =?UTF-8?q?=20function=5Fname=20and=20initialize=20methods=20abstract=20Th?= =?UTF-8?q?is=20commit=20makes=20the=20`function=5Fname`=20and=20`initiali?= =?UTF-8?q?ze`=20methods=20abstract=20in=20the=20`CustomChain`=20and=20`Cu?= =?UTF-8?q?stomAgentExecutor`=20classes.=20This=20is=20done=20to=20enforce?= =?UTF-8?q?=20the=20implementation=20of=20these=20methods=20in=20the=20sub?= =?UTF-8?q?classes=20of=20these=20classes.=20This=20change=20improves=20th?= =?UTF-8?q?e=20code=20quality=20and=20readability=20by=20making=20the=20co?= =?UTF-8?q?de=20more=20explicit=20and=20easier=20to=20understand.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/interface/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index 3670bb8ae..fa7b39650 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -91,10 +91,12 @@ class CustomChain(Chain, ABC): """Custom chain""" @staticmethod + @abstractmethod def function_name(): return "CustomChain" @classmethod + @abstractmethod def initialize(cls, *args, **kwargs): pass @@ -109,10 +111,12 @@ class CustomAgentExecutor(AgentExecutor, ABC): """Custom chain""" @staticmethod + @abstractmethod def function_name(): return "CustomChain" @classmethod + @abstractmethod def initialize(cls, *args, **kwargs): pass From d730ca027cc7d52b0d6f25319c568f77c01808e1 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Thu, 8 Jun 2023 00:20:16 -0300 Subject: [PATCH 16/69] =?UTF-8?q?=F0=9F=94=A5=20refactor(base.py):=20remov?= =?UTF-8?q?e=20redundant=20abstractmethod=20decorators=20The=20abstractmet?= =?UTF-8?q?hod=20decorator=20is=20redundant=20in=20the=20CustomChain=20and?= =?UTF-8?q?=20CustomAgentExecutor=20classes=20as=20the=20methods=20they=20?= =?UTF-8?q?decorate=20are=20already=20defined=20as=20abstract=20in=20the?= =?UTF-8?q?=20parent=20classes.=20Removing=20these=20decorators=20improves?= =?UTF-8?q?=20code=20readability=20and=20reduces=20clutter.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/interface/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index fa7b39650..3670bb8ae 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -91,12 +91,10 @@ class CustomChain(Chain, ABC): """Custom chain""" @staticmethod - @abstractmethod def function_name(): return "CustomChain" @classmethod - @abstractmethod def initialize(cls, *args, **kwargs): pass @@ -111,12 +109,10 @@ class CustomAgentExecutor(AgentExecutor, ABC): """Custom chain""" @staticmethod - @abstractmethod def function_name(): return "CustomChain" @classmethod - @abstractmethod def initialize(cls, *args, **kwargs): pass From 8ee4b2b79e8a41c927b57514d273cf64095f49d1 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Thu, 8 Jun 2023 00:39:12 -0300 Subject: [PATCH 17/69] =?UTF-8?q?=F0=9F=90=9B=20fix(complex=5Fexample.json?= =?UTF-8?q?):=20change=20=5Ftype=20value=20from=20'zero=5Fshot'=20to=20'Ze?= =?UTF-8?q?roShotPrompt'=20to=20match=20class=20name=20The=20=5Ftype=20val?= =?UTF-8?q?ue=20was=20changed=20to=20match=20the=20class=20name,=20which?= =?UTF-8?q?=20improves=20consistency=20and=20readability=20of=20the=20code?= =?UTF-8?q?.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/data/complex_example.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/complex_example.json b/tests/data/complex_example.json index 89a3b9324..d7d0d44c6 100644 --- a/tests/data/complex_example.json +++ b/tests/data/complex_example.json @@ -352,7 +352,7 @@ "type": "str", "list": false }, - "_type": "zero_shot" + "_type": "ZeroShotPrompt" }, "description": "Prompt template for Zero Shot Agent.", "base_classes": [ From f7bdb711f184e0e06ee45240c612c643bded6547 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Fri, 9 Jun 2023 15:05:56 -0300 Subject: [PATCH 18/69] Merged frontend_node/llm from origin/dev into add_extra_fields_documentloaders --- .../langflow/template/frontend_node/llms.py | 21 ++++++++++++------- .../template/frontend_node/textsplitters.py | 2 +- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/backend/langflow/template/frontend_node/llms.py b/src/backend/langflow/template/frontend_node/llms.py index 39e82422f..ac17cf8ed 100644 --- a/src/backend/langflow/template/frontend_node/llms.py +++ b/src/backend/langflow/template/frontend_node/llms.py @@ -16,20 +16,30 @@ class LLMFrontendNode(FrontendNode): def format_azure_field(field: TemplateField): if field.name == "model_name": field.show = False # Azure uses deployment_name instead of model_name. - if field.name == "openai_api_type": + elif field.name == "openai_api_type": field.show = False field.password = False field.value = "azure" - if field.name == "openai_api_version": + elif field.name == "openai_api_version": field.password = False field.value = "2023-03-15-preview" + @staticmethod + def format_llama_field(field: TemplateField): + field.show = True + field.advanced = not field.required + @staticmethod def format_field(field: TemplateField, name: Optional[str] = None) -> None: display_names_dict = { "huggingfacehub_api_token": "HuggingFace Hub API Token", } FrontendNode.format_field(field, name) + LLMFrontendNode.format_openai_field(field) + if name and "azure" in name.lower(): + LLMFrontendNode.format_azure_field(field) + if name and "llama" in name.lower(): + LLMFrontendNode.format_llama_field(field) SHOW_FIELDS = ["repo_id"] if field.name in SHOW_FIELDS: field.show = True @@ -46,7 +56,8 @@ class LLMFrontendNode(FrontendNode): field.required = True field.show = True field.is_list = True - field.options = ["text-generation", "text2text-generation"] + field.options = ["text-generation", "text2text-generation", "summarization"] + field.value = field.options[0] field.advanced = True if display_name := display_names_dict.get(field.name): @@ -64,7 +75,3 @@ class LLMFrontendNode(FrontendNode): ]: field.advanced = False field.show = True - - LLMFrontendNode.format_openai_field(field) - if "azure" in name.lower(): - LLMFrontendNode.format_azure_field(field) diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py index b784618c0..03880379d 100644 --- a/src/backend/langflow/template/frontend_node/textsplitters.py +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -10,7 +10,7 @@ class TextSplittersFrontendNode(FrontendNode): required=True, show=True, name="documents", - ) + ) ) name = "separator" if self.template.type_name == "CharacterTextSplitter": From 8a08ebdbe700a1026f29999ae89d4cebaabe3d3b Mon Sep 17 00:00:00 2001 From: Gustavo Schaedler Date: Fri, 9 Jun 2023 20:09:25 +0100 Subject: [PATCH 19/69] Remove some commented out code + add condition for the execution of make publish --- Makefile | 23 +++++++++++++++++-- src/backend/langflow/config.yaml | 1 - src/backend/langflow/graph/vertex/base.py | 2 +- src/backend/langflow/interface/chains/base.py | 6 ----- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index baf4220ff..4ff5030c6 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,11 @@ backend: make install_backend poetry run uvicorn langflow.main:app --port 7860 --reload --log-level debug +build_and_run: + echo 'Removing dist folder' + rm -rf dist + make build && poetry run pip install dist/*.tar.gz && poetry run langflow + build_frontend: cd src/frontend && CI='' npm run build cp -r src/frontend/build src/backend/langflow/frontend @@ -78,8 +83,22 @@ else endif publish: - make build - poetry publish + @while true; do \ + read -p "Do you want to build and publish LangFlow? [y/n]: " yn; \ + case "$$yn" in \ + y|Y ) \ + echo "Building..."; \ + make build; \ + echo "Publishing..."; \ + poetry publish; \ + break ;; \ + n|N ) \ + echo "Bye!!!"; \ + break ;; \ + * ) \ + echo "Invalid response, please enter 'y' or 'n'.\n" ;; \ + esac \ + done help: @echo '----' diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index e253ed8e4..96841ba0a 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -18,7 +18,6 @@ chains: - SQLDatabaseChain - RetrievalQA - RetrievalQAWithSourcesChain - - QAWithSourcesChain - ConversationalRetrievalChain - CombineDocsChain documentloaders: diff --git a/src/backend/langflow/graph/vertex/base.py b/src/backend/langflow/graph/vertex/base.py index 31cc11968..bb6ff34dc 100644 --- a/src/backend/langflow/graph/vertex/base.py +++ b/src/backend/langflow/graph/vertex/base.py @@ -53,7 +53,7 @@ class Vertex: if "Tool" not in self.output or template_dict["_type"].islower() else template_dict["_type"] ) - # self.vertex_type = template_dict["_type"] + if self.base_type is None: for base_type, value in ALL_TYPES_DICT.items(): if self.vertex_type in value: diff --git a/src/backend/langflow/interface/chains/base.py b/src/backend/langflow/interface/chains/base.py index 054ca2775..63840cf13 100644 --- a/src/backend/langflow/interface/chains/base.py +++ b/src/backend/langflow/interface/chains/base.py @@ -60,12 +60,6 @@ class ChainCreator(LangChainTypeCreator): logger.error(f"Chain {name} not loaded: {exc}") return None - # def to_list(self) -> List[str]: - # custom_chains = list(get_custom_nodes("chains").keys()) - # default_chains = list(self.type_to_loader_dict.keys()) - - # return default_chains + custom_chains - def to_list(self) -> List[str]: names = [] for _, chain in self.type_to_loader_dict.items(): From 547e10b3741e6b1f5f30d2eb23b724d423446fbe Mon Sep 17 00:00:00 2001 From: Gustavo Schaedler Date: Fri, 9 Jun 2023 20:15:00 +0100 Subject: [PATCH 20/69] Remove condition for the execution of make publish, the execution is done by GitHub actions --- Makefile | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 4ff5030c6..6c1989ee1 100644 --- a/Makefile +++ b/Makefile @@ -83,22 +83,8 @@ else endif publish: - @while true; do \ - read -p "Do you want to build and publish LangFlow? [y/n]: " yn; \ - case "$$yn" in \ - y|Y ) \ - echo "Building..."; \ - make build; \ - echo "Publishing..."; \ - poetry publish; \ - break ;; \ - n|N ) \ - echo "Bye!!!"; \ - break ;; \ - * ) \ - echo "Invalid response, please enter 'y' or 'n'.\n" ;; \ - esac \ - done + make build + poetry publish help: @echo '----' From bfb4a8e6094fc2d3143ad18e7394623c138e93f9 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 9 Jun 2023 17:08:01 -0300 Subject: [PATCH 21/69] =?UTF-8?q?=F0=9F=94=96=20chore(base.py):=20add=20di?= =?UTF-8?q?splay=5Fname=20field=20to=20FrontendNode=20The=20`display=5Fnam?= =?UTF-8?q?e`=20field=20is=20added=20to=20the=20`FrontendNode`=20class=20t?= =?UTF-8?q?o=20allow=20for=20a=20more=20user-friendly=20name=20to=20be=20d?= =?UTF-8?q?isplayed=20in=20the=20UI.=20If=20`display=5Fname`=20is=20not=20?= =?UTF-8?q?set,=20the=20`name`=20field=20will=20be=20used=20instead.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/template/frontend_node/base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/template/frontend_node/base.py b/src/backend/langflow/template/frontend_node/base.py index 074a85499..4801da086 100644 --- a/src/backend/langflow/template/frontend_node/base.py +++ b/src/backend/langflow/template/frontend_node/base.py @@ -14,6 +14,7 @@ class FrontendNode(BaseModel): description: str base_classes: List[str] name: str = "" + display_name: str = "" def to_dict(self) -> dict: return { @@ -21,7 +22,8 @@ class FrontendNode(BaseModel): "template": self.template.to_dict(self.format_field), "description": self.description, "base_classes": self.base_classes, - } + "display_name": self.display_name or self.name, + }, } def add_extra_fields(self) -> None: From bc0776657553bd5626c75a6a5200345408688b18 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 9 Jun 2023 17:08:11 -0300 Subject: [PATCH 22/69] =?UTF-8?q?=F0=9F=94=8A=20chore(base.py):=20add=20lo?= =?UTF-8?q?gger=20to=20Graph=20class=20=F0=9F=90=9B=20fix(base.py):=20impr?= =?UTF-8?q?ove=20error=20message=20when=20invalid=20payload=20is=20receive?= =?UTF-8?q?d=20The=20logger=20is=20added=20to=20the=20Graph=20class=20to?= =?UTF-8?q?=20improve=20debugging=20capabilities.=20The=20error=20message?= =?UTF-8?q?=20when=20an=20invalid=20payload=20is=20received=20is=20improve?= =?UTF-8?q?d=20to=20include=20the=20keys=20that=20were=20found=20in=20the?= =?UTF-8?q?=20payload.=20This=20makes=20it=20easier=20to=20identify=20the?= =?UTF-8?q?=20issue=20and=20fix=20it.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/graph/graph/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/graph/graph/base.py b/src/backend/langflow/graph/graph/base.py index 5fd00d09b..3653a0ec5 100644 --- a/src/backend/langflow/graph/graph/base.py +++ b/src/backend/langflow/graph/graph/base.py @@ -10,6 +10,7 @@ from langflow.graph.vertex.types import ( ) from langflow.interface.tools.constants import FILE_TOOLS from langflow.utils import payload +from langflow.utils.logger import logger class Graph: @@ -43,7 +44,9 @@ class Graph: edges = payload["edges"] return cls(nodes, edges) except KeyError as exc: - raise ValueError("Invalid payload") from exc + raise ValueError( + f"Invalid payload. Expected keys 'nodes' and 'edges'. Found {list(payload.keys())}" + ) from exc def _build_graph(self) -> None: """Builds the graph from the nodes and edges.""" From 61207db4da7069829254f935a4c5b4c1209e6620 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 9 Jun 2023 17:17:55 -0300 Subject: [PATCH 23/69] =?UTF-8?q?=F0=9F=90=9B=20fix(App.tsx):=20change=20f?= =?UTF-8?q?etch=20URL=20to=20relative=20path=20to=20fix=20incorrect=20API?= =?UTF-8?q?=20call=20The=20fetch=20URL=20was=20incorrect=20and=20was=20cal?= =?UTF-8?q?ling=20the=20API=20with=20an=20absolute=20path=20instead=20of?= =?UTF-8?q?=20a=20relative=20path.=20This=20caused=20the=20API=20call=20to?= =?UTF-8?q?=20fail.=20The=20URL=20has=20been=20changed=20to=20a=20relative?= =?UTF-8?q?=20path=20to=20fix=20the=20issue.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/src/App.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index f883ad2ef..f68c3ea73 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -49,7 +49,7 @@ export default function App() { // Initialize state variable for the version const [version, setVersion] = useState(""); useEffect(() => { - fetch("api/v1/version") + fetch("/version") .then((res) => res.json()) .then((data) => { setVersion(data.version); From 0eaa86c5ca4c02d601192fc8cbbf2676f7e57081 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sun, 11 Jun 2023 11:14:43 -0300 Subject: [PATCH 24/69] =?UTF-8?q?=F0=9F=94=A7=20chore(=5F=5Fmain=5F=5F.py)?= =?UTF-8?q?:=20add=20support=20for=20loading=20environment=20variables=20f?= =?UTF-8?q?rom=20a=20.env=20file=20The=20`load=5Fdotenv`=20function=20from?= =?UTF-8?q?=20the=20`dotenv`=20package=20is=20now=20used=20to=20load=20env?= =?UTF-8?q?ironment=20variables=20from=20a=20`.env`=20file.=20This=20allow?= =?UTF-8?q?s=20for=20a=20more=20flexible=20configuration=20of=20the=20appl?= =?UTF-8?q?ication=20as=20environment=20variables=20can=20be=20loaded=20fr?= =?UTF-8?q?om=20a=20file=20instead=20of=20being=20hardcoded=20in=20the=20c?= =?UTF-8?q?ode.=20The=20`.env`=20file=20path=20can=20be=20specified=20via?= =?UTF-8?q?=20the=20`env=5Ffile`=20parameter.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/__main__.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/__main__.py b/src/backend/langflow/__main__.py index 4a00802c9..ff99699c4 100644 --- a/src/backend/langflow/__main__.py +++ b/src/backend/langflow/__main__.py @@ -15,7 +15,7 @@ from langflow.main import create_app from langflow.settings import settings from langflow.utils.logger import configure, logger import webbrowser - +from dotenv import load_dotenv app = typer.Typer() @@ -85,6 +85,10 @@ def serve( timeout: int = typer.Option(60, help="Worker timeout in seconds."), port: int = typer.Option(7860, help="Port to listen on."), config: str = typer.Option("config.yaml", help="Path to the configuration file."), + # .env file param + env_file: Path = typer.Option( + ".env", help="Path to the .env file containing environment variables." + ), log_level: str = typer.Option("critical", help="Logging level."), log_file: Path = typer.Option("logs/langflow.log", help="Path to the log file."), jcloud: bool = typer.Option(False, help="Deploy on Jina AI Cloud"), @@ -99,11 +103,27 @@ def serve( ): """ Run the Langflow server. + + Args: + host (str): Host to bind the server to. + workers (int): Number of worker processes. + timeout (int): Worker timeout in seconds. + port (int): Port to listen on. + config (str): Path to the configuration file. + env_file (Path): Path to the .env file containing environment variables. + log_level (str): Logging level. + log_file (Path): Path to the log file. + jcloud (bool): Deploy on Jina AI Cloud. + dev (bool): Run in development mode (may contain bugs). + path (str): Path to the frontend directory containing build files. This is for development purposes only. + open_browser (bool): Open the browser after starting the server. """ if jcloud: return serve_on_jcloud() + load_dotenv(env_file) + configure(log_level=log_level, log_file=log_file) update_settings(config, dev=dev) app = create_app() From 21440578a78e97a4e766cab0e2b383413d2c39ab Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sun, 11 Jun 2023 12:17:00 -0300 Subject: [PATCH 25/69] =?UTF-8?q?=F0=9F=94=A7=20chore(API):=20add=20/api/v?= =?UTF-8?q?1/=20prefix=20to=20API=20routes=20=E2=9C=A8=20feat(API):=20add?= =?UTF-8?q?=20getVersion=20function=20to=20retrieve=20the=20version=20of?= =?UTF-8?q?=20the=20API=20The=20API=20routes=20have=20been=20updated=20to?= =?UTF-8?q?=20include=20the=20/api/v1/=20prefix=20to=20improve=20semantics?= =?UTF-8?q?=20and=20avoid=20conflicts=20with=20other=20routes.=20The=20get?= =?UTF-8?q?Version=20function=20has=20been=20added=20to=20retrieve=20the?= =?UTF-8?q?=20version=20of=20the=20API.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/frontend/src/controllers/API/index.ts | 38 ++++++++++++++++++++--- src/frontend/vite.config.ts | 12 +++---- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/frontend/src/controllers/API/index.ts b/src/frontend/src/controllers/API/index.ts index ffa9eef72..794d3aa97 100644 --- a/src/frontend/src/controllers/API/index.ts +++ b/src/frontend/src/controllers/API/index.ts @@ -3,24 +3,52 @@ import { APIObjectType, sendAllProps } from "../../types/api/index"; import axios, { AxiosResponse } from "axios"; import { FlowType } from "../../types/flow"; +// when serving with static files +// We need to add /api/v1/ to the url in the axios calls + +/** + * Retrieves all data from the API. + * @returns {Promise>} A promise that resolves to an AxiosResponse object containing the API data. + */ export async function getAll(): Promise> { - return await axios.get(`/all`); + return await axios.get(`/api/v1/all`); } export async function sendAll(data: sendAllProps) { - return await axios.post(`/predict`, data); + return await axios.post(`/api/v1/predict`, data); } -export async function checkCode( +export async function postValidateCode( code: string ): Promise> { - return await axios.post("/validate/code", { code }); + return await axios.post("/api/v1/validate/code", { code }); +} + +export async function postValidateNode( + nodeId: string, + data: any +): Promise> { + return await axios.post(`/api/v1/validate/node/${nodeId}`, { data }); } export async function checkPrompt( template: string ): Promise> { - return await axios.post("/validate/prompt", { template }); + return await axios.post("/api/v1/validate/prompt", { template }); +} + +/** + * Retrieves the version of the API. + * @returns {Promise>} A promise that resolves to an AxiosResponse object containing the API version. + * @example + * const response = await getVersion(); + * console.log(response.data.version); + * // 0.1.0 + */ +export async function getVersion(): Promise< + AxiosResponse<{ version: string }> +> { + return await axios.get("/api/v1/version"); } export async function getExamples(): Promise { diff --git a/src/frontend/vite.config.ts b/src/frontend/vite.config.ts index b1e7bdd66..6917ecc28 100644 --- a/src/frontend/vite.config.ts +++ b/src/frontend/vite.config.ts @@ -2,11 +2,11 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react-swc"; import svgr from "vite-plugin-svgr"; const apiRoutes = [ - "/all", - "/predict", - "^/validate/*", - "^/chat/*", - "/version", + "/api/v1/all", + "/api/v1/predict", + "/api/v1/validate/*", + "/api/v1/chat/*", + "/api/v1/version", "/health", ]; @@ -19,7 +19,7 @@ const proxyTargets = apiRoutes.reduce((proxyObj, route) => { changeOrigin: true, secure: false, ws: true, - rewrite: (path) => `/api/v1${path}`, + // rewrite: (path) => `/api/v1${path}`, }; return proxyObj; }, {}); From 10fce79bc366acaae07673ba0e1a5d215ee0f1d0 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Sun, 11 Jun 2023 12:17:16 -0300 Subject: [PATCH 26/69] =?UTF-8?q?=F0=9F=90=9B=20fix(App.tsx):=20replace=20?= =?UTF-8?q?fetch=20call=20with=20getVersion=20function=20call=20to=20retri?= =?UTF-8?q?eve=20app=20version=20=E2=9C=A8=20feat(App.tsx):=20add=20getVer?= =?UTF-8?q?sion=20function=20to=20retrieve=20app=20version=20from=20API=20?= =?UTF-8?q?The=20fetch=20call=20to=20retrieve=20the=20app=20version=20has?= =?UTF-8?q?=20been=20replaced=20with=20a=20call=20to=20the=20getVersion=20?= =?UTF-8?q?function.=20This=20function=20retrieves=20the=20app=20version?= =?UTF-8?q?=20from=20the=20API.=20This=20change=20improves=20the=20code's?= =?UTF-8?q?=20readability=20and=20reduces=20the=20amount=20of=20code=20nee?= =?UTF-8?q?ded=20to=20retrieve=20the=20app=20version.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 fix(GenericNode/index.tsx): replace fetch call with postValidateNode function call to validate node ✨ feat(GenericNode/index.tsx): add postValidateNode function to validate node via API The fetch call to validate the node has been replaced with a call to the postValidateNode function. This function validates the node via the API. This change improves the code's readability and reduces the amount of code needed to validate the node. 🐛 fix(codeAreaModal/index.tsx): replace checkCode function call with postValidateCode function call to validate code ✨ feat(codeAreaModal/index.tsx): add postValidateCode function to validate code via API The checkCode function call to validate the code has been replaced with a call to the postValidateCode function. This function validates the code via the API. This change improves the code's readability and reduces the amount of code needed to validate the code. --- src/frontend/src/App.tsx | 9 ++++--- .../src/CustomNodes/GenericNode/index.tsx | 24 ++++++++----------- .../src/modals/codeAreaModal/index.tsx | 4 ++-- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index f68c3ea73..e081cf490 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -14,6 +14,7 @@ import TabsManagerComponent from "./pages/FlowPage/components/tabsManagerCompone import { ErrorBoundary } from "react-error-boundary"; import CrashErrorComponent from "./components/CrashErrorComponent"; import { TabsContext } from "./contexts/tabsContext"; +import { getVersion } from "./controllers/API"; export default function App() { let { setCurrent, setShowSideBar, setIsStackedOpen } = @@ -49,11 +50,9 @@ export default function App() { // Initialize state variable for the version const [version, setVersion] = useState(""); useEffect(() => { - fetch("/version") - .then((res) => res.json()) - .then((data) => { - setVersion(data.version); - }); + getVersion().then((response) => { + setVersion(response.data.version); + }); }, []); // Use effect hook to update alertsList when a new alert is added useEffect(() => { diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index 86eaf299d..c449296ef 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -33,6 +33,7 @@ import { NodeToolbar } from "reactflow"; import NodeToolbarComponent from "../../pages/FlowPage/components/nodeToolbarComponent"; import ShadTooltip from "../../components/ShadTooltipComponent"; +import { postValidateNode } from "../../controllers/API"; export default function GenericNode({ data, selected, @@ -62,17 +63,13 @@ export default function GenericNode({ const validateNode = useCallback( debounce(async () => { try { - const response = await fetch(`/validate/node/${data.id}`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(reactFlowInstance.toObject()), - }); + const response = await postValidateNode( + data.id, + reactFlowInstance.toObject() + ); if (response.status === 200) { - let jsonResponse = await response.json(); - let jsonResponseParsed = await JSON.parse(jsonResponse); + let jsonResponseParsed = await JSON.parse(response.data); setValidationStatus(jsonResponseParsed); } } catch (error) { @@ -148,11 +145,10 @@ export default function GenericNode({ "Validating..." ) : (
- {validationStatus.params - .split("\n") - .map((line, index) => ( -
{line}
- ))} + {validationStatus.params || + "" + .split("\n") + .map((line, index) =>
{line}
)}
) } diff --git a/src/frontend/src/modals/codeAreaModal/index.tsx b/src/frontend/src/modals/codeAreaModal/index.tsx index 405187647..e74806c7b 100644 --- a/src/frontend/src/modals/codeAreaModal/index.tsx +++ b/src/frontend/src/modals/codeAreaModal/index.tsx @@ -8,7 +8,7 @@ import "ace-builds/src-noconflict/theme-twilight"; import "ace-builds/src-noconflict/ext-language_tools"; // import "ace-builds/webpack-resolver"; import { darkContext } from "../../contexts/darkContext"; -import { checkCode } from "../../controllers/API"; +import { postValidateCode } from "../../controllers/API"; import { alertContext } from "../../contexts/alertContext"; import { TabsContext } from "../../contexts/tabsContext"; import { @@ -81,7 +81,7 @@ export default function CodeAreaModal({