jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Implemented add_extra_fields method for the documentloaders

Browse source
This commit is contained in:
Alexandre Henrique 2023-05-30 17:58:13 -03:00
commit 7edce61182
5 changed files with 156 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

132
src/backend/langflow/interface/document_loaders/base.py
View file

@ -1,134 +1,44 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode
from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
from langflow.utils.util import build_template_from_class
def build_file_path_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a file path template for a document loader."""
return {
"type": "file",
"required": True,
"show": True,
"name": name,
"value": "",
"suffixes": suffixes,
"fileTypes": fileTypes,
}
suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a file path template for a document loader."""
return TemplateField(
field_type="file",
required=True,
show=True,
name=name,
value="",
suffixes=suffixes,
fileTypes=fileTypes,
)
class DocumentLoaderCreator(LangChainTypeCreator):
type_name: str = "documentloaders"
@property
def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]:
return DocumentLoaderFrontNode
@property
def type_to_loader_dict(self) -> Dict:
return documentloaders_type_to_cls_dict
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a document loader."""
try:
signature = build_template_from_class(
return build_template_from_class(
name, documentloaders_type_to_cls_dict
)
file_path_templates = {
"AirbyteJSONLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": build_file_path_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": build_file_path_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_file_path_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_file_path_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_file_path_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": build_file_path_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_file_path_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": build_file_path_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
if name in file_path_templates:
signature["template"]["file_path"] = file_path_templates[name]
elif name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
signature["template"]["web_path"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_path",
"value": "",
"display_name": "Web Page",
}
elif name in {"GitbookLoader"}:
signature["template"]["web_page"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_page",
"value": "",
"display_name": "Web Page",
}
elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}:
signature["template"]["path"] = {
"type": "str",
"required": True,
"show": True,
"name": "path",
"value": "",
"display_name": "Web Page",
}
return signature
except ValueError as exc:
raise ValueError(f"Documment Loader {name} not found") from exc
except AttributeError as exc:

4
src/backend/langflow/template/frontend_node/__init__.py
View file

@ -7,6 +7,8 @@ from langflow.template.frontend_node import (
prompts,
tools,
vectorstores,
documentloaders,
textsplitters,
)
__all__ = [
@ -18,4 +20,6 @@ __all__ = [
"llms",
"prompts",
"vectorstores",
"documentloaders",
"textsplitters",
]

122
src/backend/langflow/template/frontend_node/documentloaders.py Normal file
View file

@ -0,0 +1,122 @@
from typing import Dict, List, Optional, Type
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class DocumentLoaderFrontNode(FrontendNode):
@staticmethod
def build_template(suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a template field for a document loader."""
return TemplateField(
field_type="file",
required=True,
show=True,
name=name,
value="",
suffixes=suffixes,
fileTypes=fileTypes,
)
def get_file_path_template(self):
return {
"AirbyteJSONLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": self.build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": self.build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": self.build_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": self.build_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": self.build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": self.build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": self.build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": self.build_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": self.build_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": self.build_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": self.build_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": self.build_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": self.build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": self.build_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
def add_extra_fields(self) -> None:
file_path_templates = self.get_file_path_template()
if self.template.type_name in file_path_templates:
self.template.add_field(file_path_templates[self.template.type_name])
elif self.template.type_name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name="web_path",
value="",
display_name="Web Page",
)
)
elif self.template.type_name in {"GitbookLoader"}:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name="web_page",
value="",
display_name="Web Page",
)
)
elif self.template.type_name in {"ReadTheDocsLoader"}:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name="path",
value="",
display_name="Web Page",
)
)

9
src/backend/langflow/template/frontend_node/textsplitters.py Normal file
View file

@ -0,0 +1,9 @@
from typing import Dict, List, Optional, Type
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class TextSplittersFrontNode(FrontendNode):
def add_extra_fields(self) -> None:
pass

1
src/backend/langflow/template/frontend_node/vectorstores.py
View file

@ -3,7 +3,6 @@ from typing import Optional
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class VectorStoreFrontendNode(FrontendNode):
def add_extra_fields(self) -> None:
if self.template.type_name == "Weaviate":