refac: fix merge and format

src/backend/langflow/api/endpoints.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any, Dict
 
 from fastapi import APIRouter, HTTPException
@@ -6,7 +7,6 @@ from langflow.api.base import Code, ValidationResponse
 from langflow.interface.run import process_graph
 from langflow.interface.types import build_langchain_types_dict
 from langflow.utils.validate import validate_code
-import logging
 
 # build router
 router = APIRouter()

src/backend/langflow/cache/utils.py

@@ -2,8 +2,9 @@ import contextlib
 import hashlib
 import json
 import os
-from pathlib import Path
 import tempfile
+from pathlib import Path
+
 import dill
 
 PREFIX = "langflow_cache"

src/backend/langflow/config.yaml

@@ -33,4 +33,13 @@ toolkits:
  - OpenAPIToolkit
  - JsonToolkit
 
+embeddings:
+  #
+
+vectorstores:
+  #
+
+documentloaders:
+  #
+
 dev: false

src/backend/langflow/graph/base.py

@@ -3,17 +3,16 @@
 #   - Defer prompts building to the last moment or when they have all the tools
 #   - Build each inner agent first, then build the outer agent
 
+import logging
 import types
 from copy import deepcopy
 from typing import Any, Dict, List
-from langflow.graph.constants import DIRECT_TYPES
 
+from langflow.graph.constants import DIRECT_TYPES
 from langflow.graph.utils import load_file
 from langflow.interface import loading
 from langflow.interface.listing import ALL_TYPES_DICT
 
-import logging
-
 logger = logging.getLogger(__name__)
 
 

src/backend/langflow/graph/utils.py

@@ -1,10 +1,11 @@
 import base64
-import json
-from typing import Any
-import re
-import yaml
 import csv
 import io
+import json
+import re
+from typing import Any
+
+import yaml
 
 
 def load_file(file_name, file_content, accepted_types) -> Any:

src/backend/langflow/interface/agents/custom.py

@@ -1,20 +1,18 @@
-from typing import Optional
+from pathlib import Path
+from typing import Any, Optional
 
 from langchain import LLMChain
 from langchain.agents import AgentExecutor, ZeroShotAgent
 from langchain.agents.agent_toolkits.json.prompt import JSON_PREFIX, JSON_SUFFIX
 from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit
-from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS
-from langchain.schema import BaseLanguageModel
-from pydantic import BaseModel
-from langchain.llms.base import BaseLLM
-from typing import Any, Optional
 from langchain.agents.agent_toolkits.pandas.base import create_pandas_dataframe_agent
-from pathlib import Path
-
 from langchain.agents.agent_toolkits.pandas.prompt import PREFIX as PANDAS_PREFIX
 from langchain.agents.agent_toolkits.pandas.prompt import SUFFIX as PANDAS_SUFFIX
+from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS
+from langchain.llms.base import BaseLLM
+from langchain.schema import BaseLanguageModel
 from langchain.tools.python.tool import PythonAstREPLTool
+from pydantic import BaseModel
 
 
 class JsonAgent(AgentExecutor):

src/backend/langflow/interface/base.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 
-from langflow.template.base import TemplateField, FrontendNode, Template
+from langflow.template.base import FrontendNode, Template, TemplateField
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 

src/backend/langflow/interface/custom_lists.py

@@ -63,3 +63,162 @@ toolkit_type_to_cls_dict: dict[str, Any] = {
 wrapper_type_to_cls_dict: dict[str, Any] = {
     wrapper.__name__: wrapper for wrapper in [requests.RequestsWrapper]
 }
+
+## Embeddings
+from langchain.embeddings import (
+    CohereEmbeddings,
+    FakeEmbeddings,
+    HuggingFaceEmbeddings,
+    HuggingFaceHubEmbeddings,
+    HuggingFaceInstructEmbeddings,
+    OpenAIEmbeddings,
+    SelfHostedEmbeddings,
+    SelfHostedHuggingFaceEmbeddings,
+    SelfHostedHuggingFaceInstructEmbeddings,
+    # SagemakerEndpointEmbeddings,
+    TensorflowHubEmbeddings,
+)
+
+embedding_type_to_cls_dict = {
+    "OpenAIEmbeddings": OpenAIEmbeddings,
+    "HuggingFaceEmbeddings": HuggingFaceEmbeddings,
+    "CohereEmbeddings": CohereEmbeddings,
+    "HuggingFaceHubEmbeddings": HuggingFaceHubEmbeddings,
+    "TensorflowHubEmbeddings": TensorflowHubEmbeddings,
+    # "SagemakerEndpointEmbeddings": SagemakerEndpointEmbeddings,
+    "HuggingFaceInstructEmbeddings": HuggingFaceInstructEmbeddings,
+    "SelfHostedEmbeddings": SelfHostedEmbeddings,
+    "SelfHostedHuggingFaceEmbeddings": SelfHostedHuggingFaceEmbeddings,
+    "SelfHostedHuggingFaceInstructEmbeddings": SelfHostedHuggingFaceInstructEmbeddings,
+    "FakeEmbeddings": FakeEmbeddings,
+}
+
+## Vector Stores
+from langchain.vectorstores import (
+    FAISS,
+    AtlasDB,
+    Chroma,
+    DeepLake,
+    ElasticVectorSearch,
+    Milvus,
+    OpenSearchVectorSearch,
+    Pinecone,
+    Qdrant,
+    VectorStore,
+    Weaviate,
+)
+
+vectorstores_type_to_cls_dict = {
+    "ElasticVectorSearch": ElasticVectorSearch,
+    "FAISS": FAISS,
+    "VectorStore": VectorStore,
+    "Pinecone": Pinecone,
+    "Weaviate": Weaviate,
+    "Qdrant": Qdrant,
+    "Milvus": Milvus,
+    "Chroma": Chroma,
+    "OpenSearchVectorSearch": OpenSearchVectorSearch,
+    "AtlasDB": AtlasDB,
+    "DeepLake": DeepLake,
+}
+
+## Document Loaders
+
+from langchain.document_loaders import (
+    AirbyteJSONLoader,
+    AZLyricsLoader,
+    CollegeConfidentialLoader,
+    CoNLLULoader,
+    CSVLoader,
+    DirectoryLoader,
+    EverNoteLoader,
+    FacebookChatLoader,
+    GCSDirectoryLoader,
+    GCSFileLoader,
+    GitbookLoader,
+    GoogleApiClient,
+    GoogleApiYoutubeLoader,
+    GoogleDriveLoader,
+    GutenbergLoader,
+    HNLoader,
+    IFixitLoader,
+    IMSDbLoader,
+    NotebookLoader,
+    NotionDirectoryLoader,
+    ObsidianLoader,
+    OnlinePDFLoader,
+    PagedPDFSplitter,
+    PDFMinerLoader,
+    PyMuPDFLoader,
+    PyPDFLoader,
+    ReadTheDocsLoader,
+    RoamLoader,
+    S3DirectoryLoader,
+    S3FileLoader,
+    SRTLoader,
+    TelegramChatLoader,
+    TextLoader,
+    UnstructuredEmailLoader,
+    UnstructuredFileIOLoader,
+    UnstructuredFileLoader,
+    UnstructuredHTMLLoader,
+    UnstructuredImageLoader,
+    UnstructuredMarkdownLoader,
+    UnstructuredPDFLoader,
+    # BSHTMLLoader,
+    UnstructuredPowerPointLoader,
+    UnstructuredURLLoader,
+    UnstructuredWordDocumentLoader,
+    WebBaseLoader,
+    YoutubeLoader,
+)
+
+documentloaders_type_to_cls_dict = {
+    "UnstructuredFileLoader": UnstructuredFileLoader,
+    "UnstructuredFileIOLoader": UnstructuredFileIOLoader,
+    "UnstructuredURLLoader": UnstructuredURLLoader,
+    "DirectoryLoader": DirectoryLoader,
+    "NotionDirectoryLoader": NotionDirectoryLoader,
+    "ReadTheDocsLoader": ReadTheDocsLoader,
+    "GoogleDriveLoader": GoogleDriveLoader,
+    "UnstructuredHTMLLoader": UnstructuredHTMLLoader,
+    # "BSHTMLLoader": BSHTMLLoader,
+    "UnstructuredPowerPointLoader": UnstructuredPowerPointLoader,
+    "UnstructuredWordDocumentLoader": UnstructuredWordDocumentLoader,
+    "UnstructuredPDFLoader": UnstructuredPDFLoader,
+    "UnstructuredImageLoader": UnstructuredImageLoader,
+    "ObsidianLoader": ObsidianLoader,
+    "UnstructuredEmailLoader": UnstructuredEmailLoader,
+    "UnstructuredMarkdownLoader": UnstructuredMarkdownLoader,
+    "RoamLoader": RoamLoader,
+    "YoutubeLoader": YoutubeLoader,
+    "S3FileLoader": S3FileLoader,
+    "TextLoader": TextLoader,
+    "HNLoader": HNLoader,
+    "GitbookLoader": GitbookLoader,
+    "S3DirectoryLoader": S3DirectoryLoader,
+    "GCSFileLoader": GCSFileLoader,
+    "GCSDirectoryLoader": GCSDirectoryLoader,
+    "WebBaseLoader": WebBaseLoader,
+    "IMSDbLoader": IMSDbLoader,
+    "AZLyricsLoader": AZLyricsLoader,
+    "CollegeConfidentialLoader": CollegeConfidentialLoader,
+    "IFixitLoader": IFixitLoader,
+    "GutenbergLoader": GutenbergLoader,
+    "PagedPDFSplitter": PagedPDFSplitter,
+    "PyPDFLoader": PyPDFLoader,
+    "EverNoteLoader": EverNoteLoader,
+    "AirbyteJSONLoader": AirbyteJSONLoader,
+    "OnlinePDFLoader": OnlinePDFLoader,
+    "PDFMinerLoader": PDFMinerLoader,
+    "PyMuPDFLoader": PyMuPDFLoader,
+    "TelegramChatLoader": TelegramChatLoader,
+    "SRTLoader": SRTLoader,
+    "FacebookChatLoader": FacebookChatLoader,
+    "NotebookLoader": NotebookLoader,
+    "CoNLLULoader": CoNLLULoader,
+    "GoogleApiYoutubeLoader": GoogleApiYoutubeLoader,
+    "GoogleApiClient": GoogleApiClient,
+    "CSVLoader": CSVLoader,
+    # "BlackboardLoader",
+}

src/backend/langflow/interface/documentLoaders/base.py

@@ -0,0 +1,31 @@
+from typing import Dict, List
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.util import build_template_from_class
+
+
+class DocumentLoaderCreator(LangChainTypeCreator):
+    type_name: str = "documentloader"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return documentloaders_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Dict | None:
+        """Get the signature of a document loader."""
+        try:
+            return build_template_from_class(name, documentloaders_type_to_cls_dict)
+        except ValueError as exc:
+            raise ValueError(f"Documment Loader {name} not found") from exc
+
+    def to_list(self) -> List[str]:
+        return [
+            documentloader.__name__
+            for documentloader in self.type_to_loader_dict.values()
+            if documentloader.__name__ in settings.documentloaders or settings.dev
+        ]
+
+
+documentloader_creator = DocumentLoaderCreator()

src/backend/langflow/interface/embeddings/base.py

@@ -0,0 +1,31 @@
+from typing import Dict, List
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import embedding_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.util import build_template_from_class
+
+
+class EmbeddingCreator(LangChainTypeCreator):
+    type_name: str = "embeddings"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return embedding_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Dict | None:
+        """Get the signature of an embedding."""
+        try:
+            return build_template_from_class(name, embedding_type_to_cls_dict)
+        except ValueError as exc:
+            raise ValueError(f"Embedding {name} not found") from exc
+
+    def to_list(self) -> List[str]:
+        return [
+            embedding.__name__
+            for embedding in self.type_to_loader_dict.values()
+            if embedding.__name__ in settings.embeddings or settings.dev
+        ]
+
+
+embedding_creator = EmbeddingCreator()

src/backend/langflow/interface/run.py

@@ -2,8 +2,8 @@ import contextlib
 import io
 import re
 from typing import Any, Dict
-from langflow.cache.utils import compute_hash, load_cache, save_cache
 
+from langflow.cache.utils import compute_hash, load_cache, save_cache
 from langflow.graph.graph import Graph
 from langflow.interface import loading
 from langflow.utils import payload

src/backend/langflow/interface/tools/base.py

@@ -20,7 +20,7 @@ from langflow.interface.tools.util import (
     get_tools_dict,
 )
 from langflow.settings import settings
-from langflow.template.base import TemplateField, Template
+from langflow.template.base import Template, TemplateField
 from langflow.utils import util
 
 TOOL_INPUTS = {

src/backend/langflow/interface/types.py

@@ -1,10 +1,13 @@
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
+from langflow.interface.documentLoaders.base import documentloader_creator
+from langflow.interface.embeddings.base import embedding_creator
 from langflow.interface.llms.base import llm_creator
 from langflow.interface.memories.base import memory_creator
 from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
+from langflow.interface.vectorStore.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
 
@@ -34,6 +37,9 @@ def build_langchain_types_dict():
         tool_creator,
         toolkits_creator,
         wrapper_creator,
+        embedding_creator,
+        vectorstore_creator,
+        documentloader_creator,
     ]
 
     all_types = {}

src/backend/langflow/interface/vectorStore/base.py

@@ -0,0 +1,31 @@
+from typing import Dict, List
+
+from langflow.interface.base import LangChainTypeCreator
+from langflow.interface.custom_lists import vectorstores_type_to_cls_dict
+from langflow.settings import settings
+from langflow.utils.util import build_template_from_class
+
+
+class VectorstoreCreator(LangChainTypeCreator):
+    type_name: str = "vectorstore"
+
+    @property
+    def type_to_loader_dict(self) -> Dict:
+        return vectorstores_type_to_cls_dict
+
+    def get_signature(self, name: str) -> Dict | None:
+        """Get the signature of an embedding."""
+        try:
+            return build_template_from_class(name, vectorstores_type_to_cls_dict)
+        except ValueError as exc:
+            raise ValueError(f"Vector Store {name} not found") from exc
+
+    def to_list(self) -> List[str]:
+        return [
+            vectorstore
+            for vectorstore in self.type_to_loader_dict.keys()
+            if vectorstore in settings.vectorstores or settings.dev
+        ]
+
+
+vectorstore_creator = VectorstoreCreator()

src/backend/langflow/settings.py

@@ -12,6 +12,9 @@ class Settings(BaseSettings):
     llms: List[str] = []
     tools: List[str] = []
     memories: List[str] = []
+    embeddings: List[str] = []
+    vectorstores: List[str] = []
+    documentloaders: List[str] = []
     wrappers: List[str] = []
     toolkits: List[str] = []
     dev: bool = False

src/backend/langflow/template/base.py

@@ -1,9 +1,10 @@
 from abc import ABC
-from typing import Any, Optional, Union, Dict
-from langflow.utils import constants
+from typing import Any, Dict, Optional, Union
 
 from pydantic import BaseModel
 
+from langflow.utils import constants
+
 
 class TemplateFieldCreator(BaseModel, ABC):
     field_type: str = "str"

src/backend/langflow/template/nodes.py

@@ -1,6 +1,6 @@
 from langchain.agents.mrkl import prompt
 
-from langflow.template.base import TemplateField, FrontendNode, Template
+from langflow.template.base import FrontendNode, Template, TemplateField
 from langflow.utils.constants import DEFAULT_PYTHON_FUNCTION
 
 

src/backend/langflow/utils/util.py

@@ -119,28 +119,30 @@ def build_template_from_class(
             docs = get_class_doc(_class)
 
             variables = {"_type": _type}
-            for class_field_items, value in _class.__fields__.items():
-                if class_field_items in ["callback_manager"]:
-                    continue
-                variables[class_field_items] = {}
-                for name_, value_ in value.__repr_args__():
-                    if name_ == "default_factory":
-                        try:
-                            variables[class_field_items][
-                                "default"
-                            ] = get_default_factory(
-                                module=_class.__base__.__module__, function=value_
-                            )
-                        except Exception:
-                            variables[class_field_items]["default"] = None
-                    elif name_ not in ["name"]:
-                        variables[class_field_items][name_] = value_
 
-                variables[class_field_items]["placeholder"] = (
-                    docs["Attributes"][class_field_items]
-                    if class_field_items in docs["Attributes"]
-                    else ""
-                )
+            if "__fields__" in _class.__dict__:
+                for class_field_items, value in _class.__fields__.items():
+                    if class_field_items in ["callback_manager"]:
+                        continue
+                    variables[class_field_items] = {}
+                    for name_, value_ in value.__repr_args__():
+                        if name_ == "default_factory":
+                            try:
+                                variables[class_field_items][
+                                    "default"
+                                ] = get_default_factory(
+                                    module=_class.__base__.__module__, function=value_
+                                )
+                            except Exception:
+                                variables[class_field_items]["default"] = None
+                        elif name_ not in ["name"]:
+                            variables[class_field_items][name_] = value_
+
+                    variables[class_field_items]["placeholder"] = (
+                        docs["Attributes"][class_field_items]
+                        if class_field_items in docs["Attributes"]
+                        else ""
+                    )
             base_classes = get_base_classes(_class)
             # Adding function to base classes to allow
             # the output to be a function