From 7edce61182473a647b4b549a1d389b2d379531b5 Mon Sep 17 00:00:00 2001 From: Alexandre Henrique Date: Tue, 30 May 2023 17:58:13 -0300 Subject: [PATCH] Implemented add_extra_fields method for the documentloaders --- .../interface/document_loaders/base.py | 132 +++--------------- .../template/frontend_node/__init__.py | 4 + .../template/frontend_node/documentloaders.py | 122 ++++++++++++++++ .../template/frontend_node/textsplitters.py | 9 ++ .../template/frontend_node/vectorstores.py | 1 - 5 files changed, 156 insertions(+), 112 deletions(-) create mode 100644 src/backend/langflow/template/frontend_node/documentloaders.py create mode 100644 src/backend/langflow/template/frontend_node/textsplitters.py diff --git a/src/backend/langflow/interface/document_loaders/base.py b/src/backend/langflow/interface/document_loaders/base.py index a13d5cd5b..484db584c 100644 --- a/src/backend/langflow/interface/document_loaders/base.py +++ b/src/backend/langflow/interface/document_loaders/base.py @@ -1,134 +1,44 @@ -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Type from langflow.interface.base import LangChainTypeCreator +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode from langflow.interface.custom_lists import documentloaders_type_to_cls_dict from langflow.settings import settings from langflow.utils.logger import logger from langflow.utils.util import build_template_from_class - def build_file_path_template( - suffixes: list, fileTypes: list, name: str = "file_path" -) -> Dict: - """Build a file path template for a document loader.""" - return { - "type": "file", - "required": True, - "show": True, - "name": name, - "value": "", - "suffixes": suffixes, - "fileTypes": fileTypes, - } - + suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a file path template for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) class DocumentLoaderCreator(LangChainTypeCreator): type_name: str = "documentloaders" + @property + def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]: + return DocumentLoaderFrontNode + @property def type_to_loader_dict(self) -> Dict: return documentloaders_type_to_cls_dict - + def get_signature(self, name: str) -> Optional[Dict]: """Get the signature of a document loader.""" try: - signature = build_template_from_class( + return build_template_from_class( name, documentloaders_type_to_cls_dict ) - - file_path_templates = { - "AirbyteJSONLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "CoNLLULoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "CSVLoader": build_file_path_template( - suffixes=[".csv"], fileTypes=["csv"] - ), - "UnstructuredEmailLoader": build_file_path_template( - suffixes=[".eml"], fileTypes=["eml"] - ), - "EverNoteLoader": build_file_path_template( - suffixes=[".xml"], fileTypes=["xml"] - ), - "FacebookChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "GutenbergLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "BSHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredHTMLLoader": build_file_path_template( - suffixes=[".html"], fileTypes=["html"] - ), - "UnstructuredImageLoader": build_file_path_template( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], - ), - "UnstructuredMarkdownLoader": build_file_path_template( - suffixes=[".md"], fileTypes=["md"] - ), - "PyPDFLoader": build_file_path_template( - suffixes=[".pdf"], fileTypes=["pdf"] - ), - "UnstructuredPowerPointLoader": build_file_path_template( - suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] - ), - "SRTLoader": build_file_path_template( - suffixes=[".srt"], fileTypes=["srt"] - ), - "TelegramChatLoader": build_file_path_template( - suffixes=[".json"], fileTypes=["json"] - ), - "TextLoader": build_file_path_template( - suffixes=[".txt"], fileTypes=["txt"] - ), - "UnstructuredWordDocumentLoader": build_file_path_template( - suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] - ), - } - - if name in file_path_templates: - signature["template"]["file_path"] = file_path_templates[name] - elif name in { - "WebBaseLoader", - "AZLyricsLoader", - "CollegeConfidentialLoader", - "HNLoader", - "IFixitLoader", - "IMSDbLoader", - }: - signature["template"]["web_path"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_path", - "value": "", - "display_name": "Web Page", - } - elif name in {"GitbookLoader"}: - signature["template"]["web_page"] = { - "type": "str", - "required": True, - "show": True, - "name": "web_page", - "value": "", - "display_name": "Web Page", - } - elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}: - signature["template"]["path"] = { - "type": "str", - "required": True, - "show": True, - "name": "path", - "value": "", - "display_name": "Web Page", - } - - return signature except ValueError as exc: raise ValueError(f"Documment Loader {name} not found") from exc except AttributeError as exc: diff --git a/src/backend/langflow/template/frontend_node/__init__.py b/src/backend/langflow/template/frontend_node/__init__.py index 1aa946d41..c36234364 100644 --- a/src/backend/langflow/template/frontend_node/__init__.py +++ b/src/backend/langflow/template/frontend_node/__init__.py @@ -7,6 +7,8 @@ from langflow.template.frontend_node import ( prompts, tools, vectorstores, + documentloaders, + textsplitters, ) __all__ = [ @@ -18,4 +20,6 @@ __all__ = [ "llms", "prompts", "vectorstores", + "documentloaders", + "textsplitters", ] diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py new file mode 100644 index 000000000..d00072ad0 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -0,0 +1,122 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class DocumentLoaderFrontNode(FrontendNode): + + @staticmethod + def build_template(suffixes: list, fileTypes: list, name: str = "file_path" + ) -> Dict: + """Build a template field for a document loader.""" + return TemplateField( + field_type="file", + required=True, + show=True, + name=name, + value="", + suffixes=suffixes, + fileTypes=fileTypes, + ) + + def get_file_path_template(self): + return { + "AirbyteJSONLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "CoNLLULoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "CSVLoader": self.build_template( + suffixes=[".csv"], fileTypes=["csv"] + ), + "UnstructuredEmailLoader": self.build_template( + suffixes=[".eml"], fileTypes=["eml"] + ), + "EverNoteLoader": self.build_template( + suffixes=[".xml"], fileTypes=["xml"] + ), + "FacebookChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "GutenbergLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "BSHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredHTMLLoader": self.build_template( + suffixes=[".html"], fileTypes=["html"] + ), + "UnstructuredImageLoader": self.build_template( + suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + ), + "UnstructuredMarkdownLoader": self.build_template( + suffixes=[".md"], fileTypes=["md"] + ), + "PyPDFLoader": self.build_template( + suffixes=[".pdf"], fileTypes=["pdf"] + ), + "UnstructuredPowerPointLoader": self.build_template( + suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"] + ), + "SRTLoader": self.build_template( + suffixes=[".srt"], fileTypes=["srt"] + ), + "TelegramChatLoader": self.build_template( + suffixes=[".json"], fileTypes=["json"] + ), + "TextLoader": self.build_template( + suffixes=[".txt"], fileTypes=["txt"] + ), + "UnstructuredWordDocumentLoader": self.build_template( + suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"] + ), + } + + def add_extra_fields(self) -> None: + file_path_templates = self.get_file_path_template() + + if self.template.type_name in file_path_templates: + self.template.add_field(file_path_templates[self.template.type_name]) + elif self.template.type_name in { + "WebBaseLoader", + "AZLyricsLoader", + "CollegeConfidentialLoader", + "HNLoader", + "IFixitLoader", + "IMSDbLoader", + }: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_path", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"GitbookLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="web_page", + value="", + display_name="Web Page", + ) + ) + elif self.template.type_name in {"ReadTheDocsLoader"}: + self.template.add_field( + TemplateField( + field_type="str", + required=True, + show=True, + name="path", + value="", + display_name="Web Page", + ) + ) \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/textsplitters.py b/src/backend/langflow/template/frontend_node/textsplitters.py new file mode 100644 index 000000000..b5f2ce637 --- /dev/null +++ b/src/backend/langflow/template/frontend_node/textsplitters.py @@ -0,0 +1,9 @@ +from typing import Dict, List, Optional, Type + +from langflow.template.field.base import TemplateField +from langflow.template.frontend_node.base import FrontendNode + +class TextSplittersFrontNode(FrontendNode): + + def add_extra_fields(self) -> None: + pass \ No newline at end of file diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index 1aefaf10c..e071c3d4c 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -3,7 +3,6 @@ from typing import Optional from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode - class VectorStoreFrontendNode(FrontendNode): def add_extra_fields(self) -> None: if self.template.type_name == "Weaviate":