fix conflicts

src/backend/langflow/cache/utils.py

@@ -1,3 +1,4 @@
+import base64
 import contextlib
 import functools
 import hashlib
@@ -10,6 +11,19 @@ from pathlib import Path
 import dill  # type: ignore
 
 
+def create_cache_folder(func):
+    def wrapper(*args, **kwargs):
+        # Get the destination folder
+        cache_path = Path(tempfile.gettempdir()) / PREFIX
+
+        # Create the destination folder if it doesn't exist
+        os.makedirs(cache_path, exist_ok=True)
+
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
 def memoize_dict(maxsize=128):
     cache = OrderedDict()
 
@@ -39,9 +53,10 @@ def memoize_dict(maxsize=128):
 PREFIX = "langflow_cache"
 
 
+@create_cache_folder
 def clear_old_cache_files(max_cache_size: int = 3):
-    cache_dir = Path(tempfile.gettempdir())
-    cache_files = list(cache_dir.glob(f"{PREFIX}_*.dill"))
+    cache_dir = Path(tempfile.gettempdir()) / PREFIX
+    cache_files = list(cache_dir.glob("*.dill"))
 
     if len(cache_files) > max_cache_size:
         cache_files_sorted_by_mtime = sorted(
@@ -84,8 +99,40 @@ def filter_json(json_data):
     return filtered_data
 
 
+@create_cache_folder
+def save_binary_file(content: str, file_name: str, accepted_types: list[str]) -> str:
+    """
+    Save a binary file to the specified folder.
+
+    Args:
+        content: The content of the file as a bytes object.
+        file_name: The name of the file, including its extension.
+
+    Returns:
+        The path to the saved file.
+    """
+    if not any(file_name.endswith(suffix) for suffix in accepted_types):
+        raise ValueError(f"File {file_name} is not accepted")
+
+    # Get the destination folder
+    cache_path = Path(tempfile.gettempdir()) / PREFIX
+
+    data = content.split(",")[1]
+    decoded_bytes = base64.b64decode(data)
+
+    # Create the full file path
+    file_path = os.path.join(cache_path, file_name)
+
+    # Save the binary content to the file
+    with open(file_path, "wb") as file:
+        file.write(decoded_bytes)
+
+    return file_path
+
+
+@create_cache_folder
 def save_cache(hash_val: str, chat_data, clean_old_cache_files: bool):
-    cache_path = Path(tempfile.gettempdir()) / f"{PREFIX}_{hash_val}.dill"
+    cache_path = Path(tempfile.gettempdir()) / PREFIX / f"{hash_val}.dill"
     with cache_path.open("wb") as cache_file:
         dill.dump(chat_data, cache_file)
 
@@ -93,8 +140,9 @@ def save_cache(hash_val: str, chat_data, clean_old_cache_files: bool):
         clear_old_cache_files()
 
 
+@create_cache_folder
 def load_cache(hash_val):
-    cache_path = Path(tempfile.gettempdir()) / f"{PREFIX}_{hash_val}.dill"
+    cache_path = Path(tempfile.gettempdir()) / PREFIX / f"{hash_val}.dill"
     if cache_path.exists():
         with cache_path.open("rb") as cache_file:
             return dill.load(cache_file)

src/backend/langflow/config.yaml

@@ -61,8 +61,31 @@ vectorstores:
   - Chroma
 
 documentloaders:
+  - AirbyteJSONLoader
+  - CoNLLULoader
+  - CSVLoader
+  - UnstructuredEmailLoader
+  - EverNoteLoader
+  - FacebookChatLoader
+  - GutenbergLoader
+  - BSHTMLLoader
+  - UnstructuredHTMLLoader
+  # - UnstructuredImageLoader # Issue with Python 3.11 (https://github.com/Unstructured-IO/unstructured-inference/issues/83)
+  - UnstructuredMarkdownLoader
+  - PyPDFLoader
+  - UnstructuredPowerPointLoader
+  - SRTLoader
+  - TelegramChatLoader
   - TextLoader
+  - UnstructuredWordDocumentLoader
   - WebBaseLoader
+  - AZLyricsLoader
+  - CollegeConfidentialLoader
+  - HNLoader
+  - IFixitLoader
+  - IMSDbLoader
+  - GitbookLoader
+  - ReadTheDocsLoader
 
 textsplitters:
   - CharacterTextSplitter

src/backend/langflow/graph/base.py

@@ -8,8 +8,8 @@ import warnings
 from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
+from langflow.cache import utils as cache_utils
 from langflow.graph.constants import DIRECT_TYPES
-from langflow.graph.utils import load_file
 from langflow.interface import loading
 from langflow.interface.listing import ALL_TYPES_DICT
 from langflow.utils.logger import logger
@@ -88,8 +88,11 @@ class Node:
                 file_name = value.get("value")
                 content = value.get("content")
                 type_to_load = value.get("suffixes")
-                loaded_dict = load_file(file_name, content, type_to_load)
-                params[key] = loaded_dict
+                file_path = cache_utils.save_binary_file(
+                    content=content, file_name=file_name, accepted_types=type_to_load
+                )
+
+                params[key] = file_path
 
             # We should check if the type is in something not
             # the opposite

src/backend/langflow/graph/utils.py

@@ -1,48 +1,4 @@
-import base64
-import csv
-import io
-import json
 import re
-from typing import Any
-
-import yaml
-
-
-def load_file(file_name, file_content, accepted_types) -> Any:
-    """Load a file from a string."""
-    # Check if the file is accepted
-    if not any(file_name.endswith(suffix) for suffix in accepted_types):
-        raise ValueError(f"File {file_name} is not accepted")
-    # Get the suffix
-    suffix = file_name.split(".")[-1]
-    # file_content == 'data:application/x-yaml;base64,b3BlbmFwaTogIjMuMC4wIg...'
-    data = file_content.split(",")[1]
-    decoded_bytes = base64.b64decode(data)
-
-    # Convert the bytes object to a string
-    decoded_string = decoded_bytes.decode("utf-8")
-    if suffix == "json":
-        # Return the json content
-        return json.loads(decoded_string)
-    elif suffix in ["yaml", "yml"]:
-        # Return the yaml content
-        loaded_yaml = yaml.load(decoded_string, Loader=yaml.FullLoader)
-        try:
-            from langchain.agents.agent_toolkits.openapi.spec import reduce_openapi_spec  # type: ignore
-
-            return reduce_openapi_spec(loaded_yaml)
-        except ImportError:
-            return loaded_yaml
-
-    elif suffix == "csv":
-        # Load the csv content
-        csv_reader = csv.DictReader(io.StringIO(decoded_string))
-        return list(csv_reader)
-    elif suffix == "txt":
-        # Return the text content
-        return decoded_string
-    else:
-        raise ValueError(f"File {file_name} is not accepted")
 
 
 def validate_prompt(prompt: str):

src/backend/langflow/interface/agents/custom.py

@@ -77,7 +77,7 @@ class CSVAgent(AgentExecutor):
     @classmethod
     def from_toolkit_and_llm(
         cls,
-        path: dict,
+        path: str,
         llm: BaseLanguageModel,
         pandas_kwargs: Optional[dict] = None,
         **kwargs: Any
@@ -85,7 +85,7 @@ class CSVAgent(AgentExecutor):
         import pandas as pd  # type: ignore
 
         _kwargs = pandas_kwargs or {}
-        df = pd.DataFrame.from_dict(path, **_kwargs)
+        df = pd.read_csv(path, **_kwargs)
 
         tools = [PythonAstREPLTool(locals={"df": df})]  # type: ignore
         prompt = ZeroShotAgent.create_prompt(

src/backend/langflow/interface/document_loaders/base.py

@@ -2,26 +2,32 @@ from typing import Dict, List, Optional
 
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
-from langflow.interface.document_loaders.custom import CUSTOM_DOCUMENTLOADERS
 from langflow.settings import settings
 from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
 
 
+def build_file_path_template(
+    suffixes: list, fileTypes: list, name: str = "file_path"
+) -> Dict:
+    """Build a file path template for a document loader."""
+    return {
+        "type": "file",
+        "required": True,
+        "show": True,
+        "name": name,
+        "value": "",
+        "suffixes": suffixes,
+        "fileTypes": fileTypes,
+    }
+
+
 class DocumentLoaderCreator(LangChainTypeCreator):
     type_name: str = "documentloaders"
 
     @property
     def type_to_loader_dict(self) -> Dict:
-        types = documentloaders_type_to_cls_dict
-
-        # Drop some types that are reimplemented with the same name
-        types.pop("TextLoader")
-
-        for name, documentloader in CUSTOM_DOCUMENTLOADERS.items():
-            types[name] = documentloader
-
-        return types
+        return documentloaders_type_to_cls_dict
 
     def get_signature(self, name: str) -> Optional[Dict]:
         """Get the signature of a document loader."""
@@ -30,24 +36,96 @@ class DocumentLoaderCreator(LangChainTypeCreator):
                 name, documentloaders_type_to_cls_dict
             )
 
-            if name == "TextLoader":
-                signature["template"]["file"] = {
-                    "type": "file",
-                    "required": True,
-                    "show": True,
-                    "name": "path",
-                    "value": "",
-                    "suffixes": [".txt"],
-                    "fileTypes": ["txt"],
-                }
-            elif name == "WebBaseLoader":
+            file_path_templates = {
+                "AirbyteJSONLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "CoNLLULoader": build_file_path_template(
+                    suffixes=[".csv"], fileTypes=["csv"]
+                ),
+                "CSVLoader": build_file_path_template(
+                    suffixes=[".csv"], fileTypes=["csv"]
+                ),
+                "UnstructuredEmailLoader": build_file_path_template(
+                    suffixes=[".eml"], fileTypes=["eml"]
+                ),
+                "EverNoteLoader": build_file_path_template(
+                    suffixes=[".xml"], fileTypes=["xml"]
+                ),
+                "FacebookChatLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "GutenbergLoader": build_file_path_template(
+                    suffixes=[".txt"], fileTypes=["txt"]
+                ),
+                "BSHTMLLoader": build_file_path_template(
+                    suffixes=[".html"], fileTypes=["html"]
+                ),
+                "UnstructuredHTMLLoader": build_file_path_template(
+                    suffixes=[".html"], fileTypes=["html"]
+                ),
+                "UnstructuredImageLoader": build_file_path_template(
+                    suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
+                    fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
+                ),
+                "UnstructuredMarkdownLoader": build_file_path_template(
+                    suffixes=[".md"], fileTypes=["md"]
+                ),
+                "PyPDFLoader": build_file_path_template(
+                    suffixes=[".pdf"], fileTypes=["pdf"]
+                ),
+                "UnstructuredPowerPointLoader": build_file_path_template(
+                    suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
+                ),
+                "SRTLoader": build_file_path_template(
+                    suffixes=[".srt"], fileTypes=["srt"]
+                ),
+                "TelegramChatLoader": build_file_path_template(
+                    suffixes=[".json"], fileTypes=["json"]
+                ),
+                "TextLoader": build_file_path_template(
+                    suffixes=[".txt"], fileTypes=["txt"]
+                ),
+                "UnstructuredWordDocumentLoader": build_file_path_template(
+                    suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
+                ),
+            }
+
+            if name in file_path_templates:
+                signature["template"]["file_path"] = file_path_templates[name]
+            elif name in {
+                "WebBaseLoader",
+                "AZLyricsLoader",
+                "CollegeConfidentialLoader",
+                "HNLoader",
+                "IFixitLoader",
+                "IMSDbLoader",
+            }:
                 signature["template"]["web_path"] = {
                     "type": "str",
                     "required": True,
                     "show": True,
                     "name": "web_path",
                     "value": "",
-                    "display_name": "Web Path",
+                    "display_name": "Web Page",
+                }
+            elif name in {"GitbookLoader"}:
+                signature["template"]["web_page"] = {
+                    "type": "str",
+                    "required": True,
+                    "show": True,
+                    "name": "web_page",
+                    "value": "",
+                    "display_name": "Web Page",
+                }
+            elif name in {"ReadTheDocsLoader"}:
+                signature["template"]["path"] = {
+                    "type": "str",
+                    "required": True,
+                    "show": True,
+                    "name": "path",
+                    "value": "",
+                    "display_name": "Web Page",
                 }
 
             return signature

src/backend/langflow/interface/document_loaders/custom.py

@@ -1,22 +0,0 @@
-"""Load text files."""
-from typing import List
-
-from langchain.docstore.document import Document
-from langchain.document_loaders.base import BaseLoader
-
-
-class TextLoader(BaseLoader):
-    """Load Text files."""
-
-    def __init__(self, file: str):
-        """Initialize with file path."""
-        self.file = file
-
-    def load(self) -> List[Document]:
-        """Load from file path."""
-        return [Document(page_content=self.file, metadata={"source": "loaded"})]
-
-
-CUSTOM_DOCUMENTLOADERS = {
-    "TextLoader": TextLoader,
-}

src/backend/langflow/interface/importing/utils.py

@@ -10,7 +10,6 @@ from langchain.chat_models.base import BaseChatModel
 from langchain.llms.base import BaseLLM
 from langchain.tools import BaseTool
 
-from langflow.interface.document_loaders.custom import CUSTOM_DOCUMENTLOADERS
 from langflow.interface.tools.util import get_tool_by_name
 
 
@@ -132,8 +131,6 @@ def import_vectorstore(vectorstore: str) -> Any:
 
 def import_documentloader(documentloader: str) -> Any:
     """Import documentloader from documentloader name"""
-    if documentloader in CUSTOM_DOCUMENTLOADERS:
-        return CUSTOM_DOCUMENTLOADERS[documentloader]
 
     return import_class(f"langchain.document_loaders.{documentloader}")
 

src/backend/langflow/interface/loading.py

@@ -22,6 +22,7 @@ from langflow.interface.agents.custom import CUSTOM_AGENTS
 from langflow.interface.importing.utils import import_by_type
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.types import get_type_list
+from langflow.interface.utils import load_file_into_dict
 from langflow.utils import util, validate
 
 
@@ -36,21 +37,25 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
     if base_type == "agents":
         # We need to initialize it differently
         return load_agent_executor(class_object, params)
-    elif node_type == "ZeroShotPrompt":
-        if "tools" not in params:
-            params["tools"] = []
-        return ZeroShotAgent.create_prompt(**params)
-
-    elif node_type == "PythonFunction":
-        # If the node_type is "PythonFunction"
-        # we need to get the function from the params
-        # which will be a str containing a python function
-        # and then we need to compile it and return the function
-        # as the instance
-        function_string = params["code"]
-        if isinstance(function_string, str):
-            return validate.eval_function(function_string)
-        raise ValueError("Function should be a string")
+    elif base_type == "prompts":
+        if node_type == "ZeroShotPrompt":
+            if "tools" not in params:
+                params["tools"] = []
+            return ZeroShotAgent.create_prompt(**params)
+    elif base_type == "tools":
+        if node_type == "JsonSpec":
+            params["dict_"] = load_file_into_dict(params.pop("path"))
+            return class_object(**params)
+        elif node_type == "PythonFunction":
+            # If the node_type is "PythonFunction"
+            # we need to get the function from the params
+            # which will be a str containing a python function
+            # and then we need to compile it and return the function
+            # as the instance
+            function_string = params["code"]
+            if isinstance(function_string, str):
+                return validate.eval_function(function_string)
+            raise ValueError("Function should be a string")
     elif base_type == "toolkits":
         loaded_toolkit = class_object(**params)
         # Check if node_type has a loader
@@ -68,8 +73,8 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
         documents = params.pop("documents")
         text_splitter = class_object(**params)
         return text_splitter.split_documents(documents)
-    else:
-        return class_object(**params)
+
+    return class_object(**params)
 
 
 def load_flow_from_json(path: str):

src/backend/langflow/interface/text_splitters/base.py

@@ -26,6 +26,15 @@ class TextSplitterCreator(LangChainTypeCreator):
                 "name": "documents",
             }
 
+            signature["template"]["separator"] = {
+                "type": "str",
+                "required": True,
+                "show": True,
+                "value": ".",
+                "name": "separator",
+                "display_name": "Separator",
+            }
+
             return signature
         except ValueError as exc:
             raise ValueError(f"Text Splitter {name} not found") from exc

src/backend/langflow/interface/tools/base.py

@@ -47,12 +47,14 @@ TOOL_INPUTS = {
         value="",
         multiline=True,
     ),
-    "dict_": TemplateField(
+    "path": TemplateField(
         field_type="file",
         required=True,
         is_list=False,
         show=True,
         value="",
+        suffixes=[".json", ".yaml", ".yml"],
+        fileTypes=["json", "yaml", "yml"],
     ),
 }
 
@@ -114,6 +116,8 @@ class ToolCreator(LangChainTypeCreator):
                 return node
         elif tool_type in FILE_TOOLS:
             params = all_tools[name]["params"]  # type: ignore
+            if tool_type == "JsonSpec":
+                params["path"] = params.pop("dict_")  # type: ignore
             base_classes += [name]
         else:
             params = []

src/backend/langflow/interface/utils.py

@@ -0,0 +1,22 @@
+import json
+import os
+
+import yaml
+
+
+def load_file_into_dict(file_path: str) -> dict:
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    file_extension = os.path.splitext(file_path)[1].lower()
+
+    if file_extension == ".json":
+        with open(file_path, "r") as json_file:
+            data = json.load(json_file)
+    elif file_extension in [".yaml", ".yml"]:
+        with open(file_path, "r") as yaml_file:
+            data = yaml.safe_load(yaml_file)
+    else:
+        raise ValueError("Unsupported file type. Please provide a JSON or YAML file.")
+
+    return data