From 2532581e020f29d638d2b3a4be70cfca1421e186 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Tue, 17 Oct 2023 16:03:23 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(base.py):=20change=20'suffix?= =?UTF-8?q?es'=20to=20'fileTypes'=20in=20Vertex=20and=20LangChainTypeCreat?= =?UTF-8?q?or=20classes=20to=20improve=20consistency=20and=20readability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🐛 fix(base.py): remove 'suffixes' attribute from TemplateFieldCreator class as it is duplicate and not used 🐛 fix(base.py): change 'suffixes' to 'file_types' in TOOL_INPUTS dictionary to improve consistency and readability 🐛 fix(base.py): remove 'suffixes' attribute from TemplateFieldCreator class as it is duplicate and not used 🐛 fix(frontend_node/base.py): remove 'suffixes' attribute from FrontendNode class as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from file_path_templates dictionary as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from file_path_templates dictionary as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from file_path_templates dictionary as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from file_path_templates dictionary as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from file_path_templates dictionary as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from build_file_field function as it is duplicate and not used 🐛 fix(frontend_node/documentloaders.py): remove 'suffixes' attribute from --- src/backend/langflow/graph/vertex/base.py | 2 +- src/backend/langflow/interface/base.py | 1 - src/backend/langflow/interface/tools/base.py | 3 +- src/backend/langflow/template/field/base.py | 3 - .../langflow/template/frontend_node/agents.py | 3 +- .../langflow/template/frontend_node/base.py | 18 ++--- .../template/frontend_node/documentloaders.py | 72 +++++++++++++------ .../template/frontend_node/embeddings.py | 3 +- .../formatter/field_formatters.py | 20 +++--- .../langflow/template/frontend_node/llms.py | 3 +- src/backend/langflow/utils/util.py | 3 +- tests/test_agents_template.py | 3 +- tests/test_frontend_nodes.py | 1 - 13 files changed, 75 insertions(+), 60 deletions(-) diff --git a/src/backend/langflow/graph/vertex/base.py b/src/backend/langflow/graph/vertex/base.py index 29cf55293..1079d29a6 100644 --- a/src/backend/langflow/graph/vertex/base.py +++ b/src/backend/langflow/graph/vertex/base.py @@ -157,7 +157,7 @@ class Vertex: # If the type is not transformable to a python base class # then we need to get the edge that connects to this node if value.get("type") == "file": - # Load the type in value.get('suffixes') using + # Load the type in value.get('fileTypes') using # what is inside value.get('content') # value.get('value') is the file name if file_path := value.get("file_path"): diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index bc9db0a93..d535838ac 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -83,7 +83,6 @@ class LangChainTypeCreator(BaseModel, ABC): show=value.get("show", True), multiline=value.get("multiline", False), value=value.get("value", None), - suffixes=value.get("suffixes", []), file_types=value.get("fileTypes", []), file_path=value.get("file_path", None), ) diff --git a/src/backend/langflow/interface/tools/base.py b/src/backend/langflow/interface/tools/base.py index 56112bf30..875915a58 100644 --- a/src/backend/langflow/interface/tools/base.py +++ b/src/backend/langflow/interface/tools/base.py @@ -54,8 +54,7 @@ TOOL_INPUTS = { is_list=False, show=True, value="", - suffixes=[".json", ".yaml", ".yml"], - file_types=["json", "yaml", "yml"], + file_types=[".json", ".yaml", ".yml"], ), } diff --git a/src/backend/langflow/template/field/base.py b/src/backend/langflow/template/field/base.py index 82394c807..5d1c53387 100644 --- a/src/backend/langflow/template/field/base.py +++ b/src/backend/langflow/template/field/base.py @@ -26,9 +26,6 @@ class TemplateFieldCreator(BaseModel, ABC): value: Any = None """The value of the field. Default is None.""" - suffixes: list[str] = [] - """List of suffixes for a file field. Default is an empty list.""" - file_types: list[str] = [] """List of file types associated with the field. Default is an empty list. (duplicate)""" diff --git a/src/backend/langflow/template/frontend_node/agents.py b/src/backend/langflow/template/frontend_node/agents.py index 1d9d44aaa..d2e3589cf 100644 --- a/src/backend/langflow/template/frontend_node/agents.py +++ b/src/backend/langflow/template/frontend_node/agents.py @@ -144,8 +144,7 @@ class CSVAgentNode(FrontendNode): show=True, name="path", value="", - suffixes=[".csv"], - file_types=["csv"], + file_types=[".csv"], ), TemplateField( field_type="BaseLanguageModel", diff --git a/src/backend/langflow/template/frontend_node/base.py b/src/backend/langflow/template/frontend_node/base.py index 35686cf6a..353b3465c 100644 --- a/src/backend/langflow/template/frontend_node/base.py +++ b/src/backend/langflow/template/frontend_node/base.py @@ -2,16 +2,13 @@ import re from collections import defaultdict from typing import ClassVar, DefaultDict, Dict, List, Optional -from pydantic import BaseModel, Field - from langflow.template.field.base import TemplateField -from langflow.template.frontend_node.constants import ( - CLASSES_TO_REMOVE, - FORCE_SHOW_FIELDS, -) +from langflow.template.frontend_node.constants import (CLASSES_TO_REMOVE, + FORCE_SHOW_FIELDS) from langflow.template.frontend_node.formatter import field_formatters from langflow.template.template.base import Template from langflow.utils import constants +from pydantic import BaseModel, Field class FieldFormatters(BaseModel): @@ -138,9 +135,12 @@ class FrontendNode(BaseModel): """Handles 'dict' type by replacing it with 'code' or 'file' based on the field name.""" if "dict" in _type.lower() and field.name == "dict_": field.field_type = "file" - field.suffixes = [".json", ".yaml", ".yml"] - field.file_types = ["json", "yaml", "yml"] - elif _type.startswith("Dict") or _type.startswith("Mapping") or _type.startswith("dict"): + field.file_types = [".json", ".yaml", ".yml"] + elif ( + _type.startswith("Dict") + or _type.startswith("Mapping") + or _type.startswith("dict") + ): field.field_type = "dict" return _type diff --git a/src/backend/langflow/template/frontend_node/documentloaders.py b/src/backend/langflow/template/frontend_node/documentloaders.py index 3b9458321..40635fed3 100644 --- a/src/backend/langflow/template/frontend_node/documentloaders.py +++ b/src/backend/langflow/template/frontend_node/documentloaders.py @@ -1,9 +1,10 @@ -from typing import ClassVar, Dict, Optional +from typing import ClassVar, Dict + from langflow.template.field.base import TemplateField from langflow.template.frontend_node.base import FrontendNode -def build_file_field(suffixes: list, fileTypes: list, name: str = "file_path") -> TemplateField: +def build_file_field(fileTypes: list, name: str = "file_path") -> TemplateField: """Build a template field for a document loader.""" return TemplateField( field_type="file", @@ -11,7 +12,6 @@ def build_file_field(suffixes: list, fileTypes: list, name: str = "file_path") - show=True, name=name, value="", - suffixes=suffixes, file_types=fileTypes, ) @@ -22,25 +22,52 @@ class DocumentLoaderFrontNode(FrontendNode): self.output_types = ["Document"] file_path_templates: ClassVar[Dict] = { - "AirbyteJSONLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]), - "CoNLLULoader": build_file_field(suffixes=[".csv"], fileTypes=["csv"]), - "CSVLoader": build_file_field(suffixes=[".csv"], fileTypes=["csv"]), - "UnstructuredEmailLoader": build_file_field(suffixes=[".eml"], fileTypes=["eml"]), - "EverNoteLoader": build_file_field(suffixes=[".xml"], fileTypes=["xml"]), - "FacebookChatLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]), - "BSHTMLLoader": build_file_field(suffixes=[".html"], fileTypes=["html"]), - "UnstructuredHTMLLoader": build_file_field(suffixes=[".html"], fileTypes=["html"]), - "UnstructuredImageLoader": build_file_field( - suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], - fileTypes=["jpg", "jpeg", "png", "gif", "bmp"], + "AirbyteJSONLoader": build_file_field( + fileTypes=[".json"], + ), + "CoNLLULoader": build_file_field( + fileTypes=[".csv"], + ), + "CSVLoader": build_file_field( + fileTypes=[".csv"], + ), + "UnstructuredEmailLoader": build_file_field( + fileTypes=[".eml"], + ), + "EverNoteLoader": build_file_field( + fileTypes=[".xml"], + ), + "FacebookChatLoader": build_file_field( + fileTypes=[".json"], + ), + "BSHTMLLoader": build_file_field( + fileTypes=[".html"], + ), + "UnstructuredHTMLLoader": build_file_field(fileTypes=[".html"]), + "UnstructuredImageLoader": build_file_field( + fileTypes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"], + ), + "UnstructuredMarkdownLoader": build_file_field( + fileTypes=[".md"], + ), + "PyPDFLoader": build_file_field( + fileTypes=[".pdf"], + ), + "UnstructuredPowerPointLoader": build_file_field( + fileTypes=[".pptx", ".ppt"], + ), + "SRTLoader": build_file_field( + fileTypes=[".srt"], + ), + "TelegramChatLoader": build_file_field( + fileTypes=[".json"], + ), + "TextLoader": build_file_field( + fileTypes=[".txt"], + ), + "UnstructuredWordDocumentLoader": build_file_field( + fileTypes=[".docx", ".doc"], ), - "UnstructuredMarkdownLoader": build_file_field(suffixes=[".md"], fileTypes=["md"]), - "PyPDFLoader": build_file_field(suffixes=[".pdf"], fileTypes=["pdf"]), - "UnstructuredPowerPointLoader": build_file_field(suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]), - "SRTLoader": build_file_field(suffixes=[".srt"], fileTypes=["srt"]), - "TelegramChatLoader": build_file_field(suffixes=[".json"], fileTypes=["json"]), - "TextLoader": build_file_field(suffixes=[".txt"], fileTypes=["txt"]), - "UnstructuredWordDocumentLoader": build_file_field(suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]), } def add_extra_fields(self) -> None: @@ -101,8 +128,7 @@ class DocumentLoaderFrontNode(FrontendNode): name="zip_path", value="", display_name="Path to zip file", - suffixes=[".zip"], - file_types=["zip"], + file_types=[".zip"], ) ) self.template.add_field( diff --git a/src/backend/langflow/template/frontend_node/embeddings.py b/src/backend/langflow/template/frontend_node/embeddings.py index 4c608fe54..6d3f43d64 100644 --- a/src/backend/langflow/template/frontend_node/embeddings.py +++ b/src/backend/langflow/template/frontend_node/embeddings.py @@ -15,8 +15,7 @@ class EmbeddingFrontendNode(FrontendNode): show=True, name="credentials", value="", - suffixes=[".json"], - file_types=["json"], + file_types=[".json"], ) ) diff --git a/src/backend/langflow/template/frontend_node/formatter/field_formatters.py b/src/backend/langflow/template/frontend_node/formatter/field_formatters.py index 35fb74512..9da1ab4d0 100644 --- a/src/backend/langflow/template/frontend_node/formatter/field_formatters.py +++ b/src/backend/langflow/template/frontend_node/formatter/field_formatters.py @@ -1,14 +1,11 @@ +import re from typing import ClassVar, Dict, Optional + from langflow.template.field.base import TemplateField from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS from langflow.template.frontend_node.formatter.base import FieldFormatter -import re - -from langflow.utils.constants import ( - ANTHROPIC_MODELS, - CHAT_OPENAI_MODELS, - OPENAI_MODELS, -) +from langflow.utils.constants import (ANTHROPIC_MODELS, CHAT_OPENAI_MODELS, + OPENAI_MODELS) class OpenAIAPIKeyFormatter(FieldFormatter): @@ -152,7 +149,10 @@ class DictCodeFileFormatter(FieldFormatter): _type = value["type"] if "dict" in _type.lower() and key == "dict_": field.field_type = "file" - field.suffixes = [".json", ".yaml", ".yml"] - field.file_types = ["json", "yaml", "yml"] - elif _type.startswith("Dict") or _type.startswith("Mapping") or _type.startswith("dict"): + field.file_types = [".json", ".yaml", ".yml"] + elif ( + _type.startswith("Dict") + or _type.startswith("Mapping") + or _type.startswith("dict") + ): field.field_type = "dict" diff --git a/src/backend/langflow/template/frontend_node/llms.py b/src/backend/langflow/template/frontend_node/llms.py index 613b2c1d0..748617e16 100644 --- a/src/backend/langflow/template/frontend_node/llms.py +++ b/src/backend/langflow/template/frontend_node/llms.py @@ -18,8 +18,7 @@ class LLMFrontendNode(FrontendNode): show=True, name="credentials", value="", - suffixes=[".json"], - file_types=["json"], + file_types=[".json"], ) ) diff --git a/src/backend/langflow/utils/util.py b/src/backend/langflow/utils/util.py index c86a5f91f..0a348601d 100644 --- a/src/backend/langflow/utils/util.py +++ b/src/backend/langflow/utils/util.py @@ -403,8 +403,7 @@ def set_dict_file_attributes(value: Dict[str, Any]) -> None: Sets the file attributes for the 'dict_' key. """ value["type"] = "file" - value["suffixes"] = [".json", ".yaml", ".yml"] - value["fileTypes"] = ["json", "yaml", "yml"] + value["fileTypes"] = [".json", ".yaml", ".yml"] def replace_default_value_with_actual(value: Dict[str, Any]) -> None: diff --git a/tests/test_agents_template.py b/tests/test_agents_template.py index f2ce4e3e2..804caeb03 100644 --- a/tests/test_agents_template.py +++ b/tests/test_agents_template.py @@ -169,8 +169,7 @@ def test_csv_agent(client: TestClient, logged_in_headers): "show": True, "multiline": False, "value": "", - "suffixes": [".csv"], - "fileTypes": ["csv"], + "fileTypes": [".csv"], "password": False, "name": "path", "type": "file", diff --git a/tests/test_frontend_nodes.py b/tests/test_frontend_nodes.py index a45339d4b..815cc13c2 100644 --- a/tests/test_frontend_nodes.py +++ b/tests/test_frontend_nodes.py @@ -32,7 +32,6 @@ def test_template_field_defaults(sample_template_field: TemplateField): assert sample_template_field.show is True assert sample_template_field.multiline is False assert sample_template_field.value is None - assert sample_template_field.suffixes == [] assert sample_template_field.file_types == [] assert sample_template_field.file_path is None assert sample_template_field.password is False