jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Implemented add_extra_fields for the documentloaders

Browse source
This commit is contained in:
Alexandre Henrique 2023-05-30 19:22:59 -03:00
commit 5291e981e2
4 changed files with 107 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/backend/langflow/interface/document_loaders/base.py
View file

@ -8,20 +8,6 @@ from langflow.settings import settings
from langflow.utils.logger import logger
from langflow.utils.util import build_template_from_class
def build_file_path_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a file path template for a document loader."""
return TemplateField(
field_type="file",
required=True,
show=True,
name=name,
value="",
suffixes=suffixes,
fileTypes=fileTypes,
)
class DocumentLoaderCreator(LangChainTypeCreator):
type_name: str = "documentloaders"

45
src/backend/langflow/interface/text_splitters/base.py
View file

@ -1,6 +1,7 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.frontend_node.textsplitters import VectorStoreFrontendNode
from langflow.interface.custom_lists import textsplitter_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
@ -10,6 +11,10 @@ from langflow.utils.util import build_template_from_class
class TextSplitterCreator(LangChainTypeCreator):
type_name: str = "textsplitters"
@property
def frontend_node_class(self) -> Type[VectorStoreFrontendNode]:
return VectorStoreFrontendNode
@property
def type_to_loader_dict(self) -> Dict:
return textsplitter_type_to_cls_dict
@ -17,43 +22,7 @@ class TextSplitterCreator(LangChainTypeCreator):
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a text splitter."""
try:
signature = build_template_from_class(name, textsplitter_type_to_cls_dict)
signature["template"]["documents"] = {
"type": "BaseLoader",
"required": True,
"show": True,
"name": "documents",
}
signature["template"]["separator"] = {
"type": "str",
"required": True,
"show": True,
"value": ".",
"name": "separator",
"display_name": "Separator",
}
signature["template"]["chunk_size"] = {
"type": "int",
"required": True,
"show": True,
"value": 4000,
"name": "chunk_size",
"display_name": "Chunk Size",
}
signature["template"]["chunk_overlap"] = {
"type": "int",
"required": True,
"show": True,
"value": 200,
"name": "chunk_overlap",
"display_name": "Chunk Overlap",
}
return signature
return build_template_from_class(name, textsplitter_type_to_cls_dict)
except ValueError as exc:
raise ValueError(f"Text Splitter {name} not found") from exc
except AttributeError as exc:

123
src/backend/langflow/template/frontend_node/documentloaders.py
View file

@ -7,7 +7,7 @@ class DocumentLoaderFrontNode(FrontendNode):
@staticmethod
def build_template(suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
) -> TemplateField:
"""Build a template field for a document loader."""
return TemplateField(
field_type="file",
@ -18,68 +18,65 @@ class DocumentLoaderFrontNode(FrontendNode):
suffixes=suffixes,
fileTypes=fileTypes,
)
def get_file_path_template(self):
return {
"AirbyteJSONLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": self.build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": self.build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": self.build_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": self.build_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": self.build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": self.build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": self.build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": self.build_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": self.build_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": self.build_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": self.build_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": self.build_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": self.build_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": self.build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": self.build_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
file_path_templates = {
"AirbyteJSONLoader": build_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": build_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": build_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": build_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": build_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": build_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": build_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": build_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": build_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
def add_extra_fields(self) -> None:
file_path_templates = self.get_file_path_template()
if self.template.type_name in file_path_templates:
self.template.add_field(file_path_templates[self.template.type_name])
if self.template.type_name in self.file_path_templates:
self.template.add_field(self.file_path_templates[self.template.type_name])
elif self.template.type_name in {
"WebBaseLoader",
"AZLyricsLoader",
@ -119,4 +116,6 @@ class DocumentLoaderFrontNode(FrontendNode):
value="",
display_name="Web Page",
)
)
)

41
src/backend/langflow/template/frontend_node/textsplitters.py
View file

@ -3,7 +3,44 @@ from typing import Dict, List, Optional, Type
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class TextSplittersFrontNode(FrontendNode):
class VectorStoreFrontendNode(FrontendNode):
def add_extra_fields(self) -> None:
pass
pass
"""
signature["template"]["documents"] = {
"type": "BaseLoader",
"required": True,
"show": True,
"name": "documents",
}
signature["template"]["separator"] = {
"type": "str",
"required": True,
"show": True,
"value": ".",
"name": "separator",
"display_name": "Separator",
}
signature["template"]["chunk_size"] = {
"type": "int",
"required": True,
"show": True,
"value": 4000,
"name": "chunk_size",
"display_name": "Chunk Size",
}
signature["template"]["chunk_overlap"] = {
"type": "int",
"required": True,
"show": True,
"value": 200,
"name": "chunk_overlap",
"display_name": "Chunk Overlap",
}
"""