diff --git a/src/backend/langflow/api/endpoints.py b/src/backend/langflow/api/endpoints.py index 41089e699..c82616d3e 100644 --- a/src/backend/langflow/api/endpoints.py +++ b/src/backend/langflow/api/endpoints.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Dict from fastapi import APIRouter, HTTPException @@ -6,7 +7,6 @@ from langflow.api.base import Code, ValidationResponse from langflow.interface.run import process_graph from langflow.interface.types import build_langchain_types_dict from langflow.utils.validate import validate_code -import logging # build router router = APIRouter() diff --git a/src/backend/langflow/cache/utils.py b/src/backend/langflow/cache/utils.py index 2c4acd074..1d1f31de1 100644 --- a/src/backend/langflow/cache/utils.py +++ b/src/backend/langflow/cache/utils.py @@ -2,8 +2,9 @@ import contextlib import hashlib import json import os -from pathlib import Path import tempfile +from pathlib import Path + import dill PREFIX = "langflow_cache" diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 07d7218cb..5f2d37d74 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -33,4 +33,13 @@ toolkits: - OpenAPIToolkit - JsonToolkit +embeddings: + # + +vectorstores: + # + +documentloaders: + # + dev: false diff --git a/src/backend/langflow/graph/base.py b/src/backend/langflow/graph/base.py index f4b41bbfc..08ad786ac 100644 --- a/src/backend/langflow/graph/base.py +++ b/src/backend/langflow/graph/base.py @@ -3,17 +3,16 @@ # - Defer prompts building to the last moment or when they have all the tools # - Build each inner agent first, then build the outer agent +import logging import types from copy import deepcopy from typing import Any, Dict, List -from langflow.graph.constants import DIRECT_TYPES +from langflow.graph.constants import DIRECT_TYPES from langflow.graph.utils import load_file from langflow.interface import loading from langflow.interface.listing import ALL_TYPES_DICT -import logging - logger = logging.getLogger(__name__) diff --git a/src/backend/langflow/graph/utils.py b/src/backend/langflow/graph/utils.py index ca728390d..e7dc9e593 100644 --- a/src/backend/langflow/graph/utils.py +++ b/src/backend/langflow/graph/utils.py @@ -1,10 +1,11 @@ import base64 -import json -from typing import Any -import re -import yaml import csv import io +import json +import re +from typing import Any + +import yaml def load_file(file_name, file_content, accepted_types) -> Any: diff --git a/src/backend/langflow/interface/agents/custom.py b/src/backend/langflow/interface/agents/custom.py index c056b7c72..64682ef4a 100644 --- a/src/backend/langflow/interface/agents/custom.py +++ b/src/backend/langflow/interface/agents/custom.py @@ -1,20 +1,18 @@ -from typing import Optional +from pathlib import Path +from typing import Any, Optional from langchain import LLMChain from langchain.agents import AgentExecutor, ZeroShotAgent from langchain.agents.agent_toolkits.json.prompt import JSON_PREFIX, JSON_SUFFIX from langchain.agents.agent_toolkits.json.toolkit import JsonToolkit -from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS -from langchain.schema import BaseLanguageModel -from pydantic import BaseModel -from langchain.llms.base import BaseLLM -from typing import Any, Optional from langchain.agents.agent_toolkits.pandas.base import create_pandas_dataframe_agent -from pathlib import Path - from langchain.agents.agent_toolkits.pandas.prompt import PREFIX as PANDAS_PREFIX from langchain.agents.agent_toolkits.pandas.prompt import SUFFIX as PANDAS_SUFFIX +from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS +from langchain.llms.base import BaseLLM +from langchain.schema import BaseLanguageModel from langchain.tools.python.tool import PythonAstREPLTool +from pydantic import BaseModel class JsonAgent(AgentExecutor): diff --git a/src/backend/langflow/interface/base.py b/src/backend/langflow/interface/base.py index 2ec7bc12e..eb3432d12 100644 --- a/src/backend/langflow/interface/base.py +++ b/src/backend/langflow/interface/base.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from pydantic import BaseModel -from langflow.template.base import TemplateField, FrontendNode, Template +from langflow.template.base import FrontendNode, Template, TemplateField # Assuming necessary imports for Field, Template, and FrontendNode classes diff --git a/src/backend/langflow/interface/custom_lists.py b/src/backend/langflow/interface/custom_lists.py index 91a62d651..c4522848a 100644 --- a/src/backend/langflow/interface/custom_lists.py +++ b/src/backend/langflow/interface/custom_lists.py @@ -63,3 +63,162 @@ toolkit_type_to_cls_dict: dict[str, Any] = { wrapper_type_to_cls_dict: dict[str, Any] = { wrapper.__name__: wrapper for wrapper in [requests.RequestsWrapper] } + +## Embeddings +from langchain.embeddings import ( + CohereEmbeddings, + FakeEmbeddings, + HuggingFaceEmbeddings, + HuggingFaceHubEmbeddings, + HuggingFaceInstructEmbeddings, + OpenAIEmbeddings, + SelfHostedEmbeddings, + SelfHostedHuggingFaceEmbeddings, + SelfHostedHuggingFaceInstructEmbeddings, + # SagemakerEndpointEmbeddings, + TensorflowHubEmbeddings, +) + +embedding_type_to_cls_dict = { + "OpenAIEmbeddings": OpenAIEmbeddings, + "HuggingFaceEmbeddings": HuggingFaceEmbeddings, + "CohereEmbeddings": CohereEmbeddings, + "HuggingFaceHubEmbeddings": HuggingFaceHubEmbeddings, + "TensorflowHubEmbeddings": TensorflowHubEmbeddings, + # "SagemakerEndpointEmbeddings": SagemakerEndpointEmbeddings, + "HuggingFaceInstructEmbeddings": HuggingFaceInstructEmbeddings, + "SelfHostedEmbeddings": SelfHostedEmbeddings, + "SelfHostedHuggingFaceEmbeddings": SelfHostedHuggingFaceEmbeddings, + "SelfHostedHuggingFaceInstructEmbeddings": SelfHostedHuggingFaceInstructEmbeddings, + "FakeEmbeddings": FakeEmbeddings, +} + +## Vector Stores +from langchain.vectorstores import ( + FAISS, + AtlasDB, + Chroma, + DeepLake, + ElasticVectorSearch, + Milvus, + OpenSearchVectorSearch, + Pinecone, + Qdrant, + VectorStore, + Weaviate, +) + +vectorstores_type_to_cls_dict = { + "ElasticVectorSearch": ElasticVectorSearch, + "FAISS": FAISS, + "VectorStore": VectorStore, + "Pinecone": Pinecone, + "Weaviate": Weaviate, + "Qdrant": Qdrant, + "Milvus": Milvus, + "Chroma": Chroma, + "OpenSearchVectorSearch": OpenSearchVectorSearch, + "AtlasDB": AtlasDB, + "DeepLake": DeepLake, +} + +## Document Loaders + +from langchain.document_loaders import ( + AirbyteJSONLoader, + AZLyricsLoader, + CollegeConfidentialLoader, + CoNLLULoader, + CSVLoader, + DirectoryLoader, + EverNoteLoader, + FacebookChatLoader, + GCSDirectoryLoader, + GCSFileLoader, + GitbookLoader, + GoogleApiClient, + GoogleApiYoutubeLoader, + GoogleDriveLoader, + GutenbergLoader, + HNLoader, + IFixitLoader, + IMSDbLoader, + NotebookLoader, + NotionDirectoryLoader, + ObsidianLoader, + OnlinePDFLoader, + PagedPDFSplitter, + PDFMinerLoader, + PyMuPDFLoader, + PyPDFLoader, + ReadTheDocsLoader, + RoamLoader, + S3DirectoryLoader, + S3FileLoader, + SRTLoader, + TelegramChatLoader, + TextLoader, + UnstructuredEmailLoader, + UnstructuredFileIOLoader, + UnstructuredFileLoader, + UnstructuredHTMLLoader, + UnstructuredImageLoader, + UnstructuredMarkdownLoader, + UnstructuredPDFLoader, + # BSHTMLLoader, + UnstructuredPowerPointLoader, + UnstructuredURLLoader, + UnstructuredWordDocumentLoader, + WebBaseLoader, + YoutubeLoader, +) + +documentloaders_type_to_cls_dict = { + "UnstructuredFileLoader": UnstructuredFileLoader, + "UnstructuredFileIOLoader": UnstructuredFileIOLoader, + "UnstructuredURLLoader": UnstructuredURLLoader, + "DirectoryLoader": DirectoryLoader, + "NotionDirectoryLoader": NotionDirectoryLoader, + "ReadTheDocsLoader": ReadTheDocsLoader, + "GoogleDriveLoader": GoogleDriveLoader, + "UnstructuredHTMLLoader": UnstructuredHTMLLoader, + # "BSHTMLLoader": BSHTMLLoader, + "UnstructuredPowerPointLoader": UnstructuredPowerPointLoader, + "UnstructuredWordDocumentLoader": UnstructuredWordDocumentLoader, + "UnstructuredPDFLoader": UnstructuredPDFLoader, + "UnstructuredImageLoader": UnstructuredImageLoader, + "ObsidianLoader": ObsidianLoader, + "UnstructuredEmailLoader": UnstructuredEmailLoader, + "UnstructuredMarkdownLoader": UnstructuredMarkdownLoader, + "RoamLoader": RoamLoader, + "YoutubeLoader": YoutubeLoader, + "S3FileLoader": S3FileLoader, + "TextLoader": TextLoader, + "HNLoader": HNLoader, + "GitbookLoader": GitbookLoader, + "S3DirectoryLoader": S3DirectoryLoader, + "GCSFileLoader": GCSFileLoader, + "GCSDirectoryLoader": GCSDirectoryLoader, + "WebBaseLoader": WebBaseLoader, + "IMSDbLoader": IMSDbLoader, + "AZLyricsLoader": AZLyricsLoader, + "CollegeConfidentialLoader": CollegeConfidentialLoader, + "IFixitLoader": IFixitLoader, + "GutenbergLoader": GutenbergLoader, + "PagedPDFSplitter": PagedPDFSplitter, + "PyPDFLoader": PyPDFLoader, + "EverNoteLoader": EverNoteLoader, + "AirbyteJSONLoader": AirbyteJSONLoader, + "OnlinePDFLoader": OnlinePDFLoader, + "PDFMinerLoader": PDFMinerLoader, + "PyMuPDFLoader": PyMuPDFLoader, + "TelegramChatLoader": TelegramChatLoader, + "SRTLoader": SRTLoader, + "FacebookChatLoader": FacebookChatLoader, + "NotebookLoader": NotebookLoader, + "CoNLLULoader": CoNLLULoader, + "GoogleApiYoutubeLoader": GoogleApiYoutubeLoader, + "GoogleApiClient": GoogleApiClient, + "CSVLoader": CSVLoader, + # "BlackboardLoader", +} diff --git a/src/backend/langflow/interface/documentLoaders/base.py b/src/backend/langflow/interface/documentLoaders/base.py new file mode 100644 index 000000000..494fabfd3 --- /dev/null +++ b/src/backend/langflow/interface/documentLoaders/base.py @@ -0,0 +1,31 @@ +from typing import Dict, List + +from langflow.interface.base import LangChainTypeCreator +from langflow.interface.custom_lists import documentloaders_type_to_cls_dict +from langflow.settings import settings +from langflow.utils.util import build_template_from_class + + +class DocumentLoaderCreator(LangChainTypeCreator): + type_name: str = "documentloader" + + @property + def type_to_loader_dict(self) -> Dict: + return documentloaders_type_to_cls_dict + + def get_signature(self, name: str) -> Dict | None: + """Get the signature of a document loader.""" + try: + return build_template_from_class(name, documentloaders_type_to_cls_dict) + except ValueError as exc: + raise ValueError(f"Documment Loader {name} not found") from exc + + def to_list(self) -> List[str]: + return [ + documentloader.__name__ + for documentloader in self.type_to_loader_dict.values() + if documentloader.__name__ in settings.documentloaders or settings.dev + ] + + +documentloader_creator = DocumentLoaderCreator() diff --git a/src/backend/langflow/interface/embeddings/base.py b/src/backend/langflow/interface/embeddings/base.py new file mode 100644 index 000000000..de1921b5e --- /dev/null +++ b/src/backend/langflow/interface/embeddings/base.py @@ -0,0 +1,31 @@ +from typing import Dict, List + +from langflow.interface.base import LangChainTypeCreator +from langflow.interface.custom_lists import embedding_type_to_cls_dict +from langflow.settings import settings +from langflow.utils.util import build_template_from_class + + +class EmbeddingCreator(LangChainTypeCreator): + type_name: str = "embeddings" + + @property + def type_to_loader_dict(self) -> Dict: + return embedding_type_to_cls_dict + + def get_signature(self, name: str) -> Dict | None: + """Get the signature of an embedding.""" + try: + return build_template_from_class(name, embedding_type_to_cls_dict) + except ValueError as exc: + raise ValueError(f"Embedding {name} not found") from exc + + def to_list(self) -> List[str]: + return [ + embedding.__name__ + for embedding in self.type_to_loader_dict.values() + if embedding.__name__ in settings.embeddings or settings.dev + ] + + +embedding_creator = EmbeddingCreator() diff --git a/src/backend/langflow/interface/run.py b/src/backend/langflow/interface/run.py index 998ee297d..cb86789ae 100644 --- a/src/backend/langflow/interface/run.py +++ b/src/backend/langflow/interface/run.py @@ -2,8 +2,8 @@ import contextlib import io import re from typing import Any, Dict -from langflow.cache.utils import compute_hash, load_cache, save_cache +from langflow.cache.utils import compute_hash, load_cache, save_cache from langflow.graph.graph import Graph from langflow.interface import loading from langflow.utils import payload diff --git a/src/backend/langflow/interface/tools/base.py b/src/backend/langflow/interface/tools/base.py index 8d14e3612..979e7b8fb 100644 --- a/src/backend/langflow/interface/tools/base.py +++ b/src/backend/langflow/interface/tools/base.py @@ -20,7 +20,7 @@ from langflow.interface.tools.util import ( get_tools_dict, ) from langflow.settings import settings -from langflow.template.base import TemplateField, Template +from langflow.template.base import Template, TemplateField from langflow.utils import util TOOL_INPUTS = { diff --git a/src/backend/langflow/interface/types.py b/src/backend/langflow/interface/types.py index 2db27bfc3..aaa616de3 100644 --- a/src/backend/langflow/interface/types.py +++ b/src/backend/langflow/interface/types.py @@ -1,10 +1,13 @@ from langflow.interface.agents.base import agent_creator from langflow.interface.chains.base import chain_creator +from langflow.interface.documentLoaders.base import documentloader_creator +from langflow.interface.embeddings.base import embedding_creator from langflow.interface.llms.base import llm_creator from langflow.interface.memories.base import memory_creator from langflow.interface.prompts.base import prompt_creator from langflow.interface.toolkits.base import toolkits_creator from langflow.interface.tools.base import tool_creator +from langflow.interface.vectorStore.base import vectorstore_creator from langflow.interface.wrappers.base import wrapper_creator @@ -34,6 +37,9 @@ def build_langchain_types_dict(): tool_creator, toolkits_creator, wrapper_creator, + embedding_creator, + vectorstore_creator, + documentloader_creator, ] all_types = {} diff --git a/src/backend/langflow/interface/vectorStore/base.py b/src/backend/langflow/interface/vectorStore/base.py new file mode 100644 index 000000000..01c222e07 --- /dev/null +++ b/src/backend/langflow/interface/vectorStore/base.py @@ -0,0 +1,31 @@ +from typing import Dict, List + +from langflow.interface.base import LangChainTypeCreator +from langflow.interface.custom_lists import vectorstores_type_to_cls_dict +from langflow.settings import settings +from langflow.utils.util import build_template_from_class + + +class VectorstoreCreator(LangChainTypeCreator): + type_name: str = "vectorstore" + + @property + def type_to_loader_dict(self) -> Dict: + return vectorstores_type_to_cls_dict + + def get_signature(self, name: str) -> Dict | None: + """Get the signature of an embedding.""" + try: + return build_template_from_class(name, vectorstores_type_to_cls_dict) + except ValueError as exc: + raise ValueError(f"Vector Store {name} not found") from exc + + def to_list(self) -> List[str]: + return [ + vectorstore + for vectorstore in self.type_to_loader_dict.keys() + if vectorstore in settings.vectorstores or settings.dev + ] + + +vectorstore_creator = VectorstoreCreator() diff --git a/src/backend/langflow/settings.py b/src/backend/langflow/settings.py index a8c2edc4a..c0fd66c58 100644 --- a/src/backend/langflow/settings.py +++ b/src/backend/langflow/settings.py @@ -12,6 +12,9 @@ class Settings(BaseSettings): llms: List[str] = [] tools: List[str] = [] memories: List[str] = [] + embeddings: List[str] = [] + vectorstores: List[str] = [] + documentloaders: List[str] = [] wrappers: List[str] = [] toolkits: List[str] = [] dev: bool = False diff --git a/src/backend/langflow/template/base.py b/src/backend/langflow/template/base.py index 5365b96c0..eb7b8898b 100644 --- a/src/backend/langflow/template/base.py +++ b/src/backend/langflow/template/base.py @@ -1,9 +1,10 @@ from abc import ABC -from typing import Any, Optional, Union, Dict -from langflow.utils import constants +from typing import Any, Dict, Optional, Union from pydantic import BaseModel +from langflow.utils import constants + class TemplateFieldCreator(BaseModel, ABC): field_type: str = "str" diff --git a/src/backend/langflow/template/nodes.py b/src/backend/langflow/template/nodes.py index 96219e000..718d66387 100644 --- a/src/backend/langflow/template/nodes.py +++ b/src/backend/langflow/template/nodes.py @@ -1,6 +1,6 @@ from langchain.agents.mrkl import prompt -from langflow.template.base import TemplateField, FrontendNode, Template +from langflow.template.base import FrontendNode, Template, TemplateField from langflow.utils.constants import DEFAULT_PYTHON_FUNCTION diff --git a/src/backend/langflow/utils/util.py b/src/backend/langflow/utils/util.py index 2f6b66a43..59a19eb33 100644 --- a/src/backend/langflow/utils/util.py +++ b/src/backend/langflow/utils/util.py @@ -119,28 +119,30 @@ def build_template_from_class( docs = get_class_doc(_class) variables = {"_type": _type} - for class_field_items, value in _class.__fields__.items(): - if class_field_items in ["callback_manager"]: - continue - variables[class_field_items] = {} - for name_, value_ in value.__repr_args__(): - if name_ == "default_factory": - try: - variables[class_field_items][ - "default" - ] = get_default_factory( - module=_class.__base__.__module__, function=value_ - ) - except Exception: - variables[class_field_items]["default"] = None - elif name_ not in ["name"]: - variables[class_field_items][name_] = value_ - variables[class_field_items]["placeholder"] = ( - docs["Attributes"][class_field_items] - if class_field_items in docs["Attributes"] - else "" - ) + if "__fields__" in _class.__dict__: + for class_field_items, value in _class.__fields__.items(): + if class_field_items in ["callback_manager"]: + continue + variables[class_field_items] = {} + for name_, value_ in value.__repr_args__(): + if name_ == "default_factory": + try: + variables[class_field_items][ + "default" + ] = get_default_factory( + module=_class.__base__.__module__, function=value_ + ) + except Exception: + variables[class_field_items]["default"] = None + elif name_ not in ["name"]: + variables[class_field_items][name_] = value_ + + variables[class_field_items]["placeholder"] = ( + docs["Attributes"][class_field_items] + if class_field_items in docs["Attributes"] + else "" + ) base_classes = get_base_classes(_class) # Adding function to base classes to allow # the output to be a function diff --git a/tests/test_cache.py b/tests/test_cache.py index 0d9102b49..04c359ac4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -1,11 +1,12 @@ -import json import hashlib -from pathlib import Path -import dill +import json import tempfile -from langflow.cache.utils import compute_hash, load_cache, save_cache, PREFIX -from langflow.interface.run import load_langchain_object, process_graph +from pathlib import Path + +import dill import pytest +from langflow.cache.utils import PREFIX, compute_hash, load_cache, save_cache +from langflow.interface.run import load_langchain_object, process_graph def get_graph(_type="basic"): diff --git a/tests/test_creators.py b/tests/test_creators.py index 08b2fc9d1..42ee2fbca 100644 --- a/tests/test_creators.py +++ b/tests/test_creators.py @@ -1,7 +1,8 @@ from typing import Dict, List -from langflow.interface.base import LangChainTypeCreator -from langflow.interface.agents.base import AgentCreator + import pytest +from langflow.interface.agents.base import AgentCreator +from langflow.interface.base import LangChainTypeCreator @pytest.fixture diff --git a/tests/test_endpoints.py b/tests/test_endpoints.py index e6eb8b85c..570bf554e 100644 --- a/tests/test_endpoints.py +++ b/tests/test_endpoints.py @@ -1,5 +1,5 @@ -from langflow.interface.tools.constants import CUSTOM_TOOLS from fastapi.testclient import TestClient +from langflow.interface.tools.constants import CUSTOM_TOOLS def test_get_all(client: TestClient): diff --git a/tests/test_frontend_nodes.py b/tests/test_frontend_nodes.py index 75b0b741f..652da90db 100644 --- a/tests/test_frontend_nodes.py +++ b/tests/test_frontend_nodes.py @@ -1,8 +1,9 @@ -import pytest from typing import Dict, List -from langflow.template.base import TemplateField, FrontendNode, Template -from langflow.interface.base import LangChainTypeCreator + +import pytest from langflow.interface.agents.base import AgentCreator +from langflow.interface.base import LangChainTypeCreator +from langflow.template.base import FrontendNode, Template, TemplateField @pytest.fixture diff --git a/tests/test_graph.py b/tests/test_graph.py index 9bc7b4220..0ea2d0f51 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -1,21 +1,20 @@ import json -from langflow.graph.nodes import ( - WrapperNode, - AgentNode, - ToolNode, - ChainNode, - PromptNode, - LLMNode, - ToolkitNode, - FileToolNode, -) import pytest from langchain.agents import AgentExecutor from langflow.graph import Edge, Graph, Node +from langflow.graph.nodes import ( + AgentNode, + ChainNode, + FileToolNode, + LLMNode, + PromptNode, + ToolkitNode, + ToolNode, + WrapperNode, +) from langflow.utils.payload import build_json, get_root_node - # Test cases for the graph module # now we have three types of graph: diff --git a/tests/test_template.py b/tests/test_template.py index ed22bae3d..f557256e0 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,17 +1,18 @@ -from langflow.utils.constants import CHAT_OPENAI_MODELS, OPENAI_MODELS -from pydantic import BaseModel -import pytest -import re import importlib +import re from typing import Dict, List, Optional + +import pytest +from langflow.utils.constants import CHAT_OPENAI_MODELS, OPENAI_MODELS from langflow.utils.util import ( build_template_from_class, build_template_from_function, format_dict, get_base_classes, - get_default_factory, get_class_doc, + get_default_factory, ) +from pydantic import BaseModel # Dummy classes for testing purposes