jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/NewId' into db

Browse source
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-06-14 14:56:46 -03:00
commit 010c1b7b50
40 changed files with 476 additions and 291 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -51,7 +51,6 @@ build_and_run:
rm -rf dist
make build && poetry run pip install dist/*.tar.gz && poetry run langflow
build_frontend:
cd src/frontend && CI='' npm run build
cp -r src/frontend/build src/backend/langflow/frontend

8
src/backend/langflow/__init__.py
View file

@ -1,4 +1,12 @@
from importlib import metadata
from langflow.cache import cache_manager
from langflow.processing.process import load_flow_from_json
try:
__version__ = metadata.version(__package__)
except metadata.PackageNotFoundError:
# Case where package metadata is not available.
__version__ = ""
del metadata # optional, avoids polluting the results of dir(__package__)
__all__ = ["load_flow_from_json", "cache_manager"]

22
src/backend/langflow/__main__.py
View file

@ -16,7 +16,7 @@ from langflow.main import create_app
from langflow.settings import settings
from langflow.utils.logger import configure, logger
import webbrowser
from dotenv import load_dotenv
app = typer.Typer()
@ -88,6 +88,10 @@ def serve(
timeout: int = typer.Option(60, help="Worker timeout in seconds."),
port: int = typer.Option(7860, help="Port to listen on."),
config: str = typer.Option("config.yaml", help="Path to the configuration file."),
# .env file param
env_file: Path = typer.Option(
".env", help="Path to the .env file containing environment variables."
),
log_level: str = typer.Option("critical", help="Logging level."),
log_file: Path = typer.Option("logs/langflow.log", help="Path to the log file."),
jcloud: bool = typer.Option(False, help="Deploy on Jina AI Cloud"),
@ -106,11 +110,27 @@ def serve(
):
"""
Run the Langflow server.
Args:
host (str): Host to bind the server to.
workers (int): Number of worker processes.
timeout (int): Worker timeout in seconds.
port (int): Port to listen on.
config (str): Path to the configuration file.
env_file (Path): Path to the .env file containing environment variables.
log_level (str): Logging level.
log_file (Path): Path to the log file.
jcloud (bool): Deploy on Jina AI Cloud.
dev (bool): Run in development mode (may contain bugs).
path (str): Path to the frontend directory containing build files. This is for development purposes only.
open_browser (bool): Open the browser after starting the server.
"""
if jcloud:
return serve_on_jcloud()
load_dotenv(env_file)
configure(log_level=log_level, log_file=log_file)
update_settings(config, dev=dev, database_url=database_url)
app = create_app()

4
src/backend/langflow/api/v1/endpoints.py
View file

@ -70,4 +70,6 @@ async def predict_flow(
# get endpoint to return version of langflow
@router.get("/version")
def get_version():
return {"version": version("langflow")}
from langflow import __version__
return {"version": __version__}

8
src/backend/langflow/config.yaml
View file

@ -16,6 +16,10 @@ chains:
- MidJourneyPromptChain
- TimeTravelGuideChain
- SQLDatabaseChain
- RetrievalQA
- RetrievalQAWithSourcesChain
- ConversationalRetrievalChain
- CombineDocsChain
documentloaders:
- AirbyteJSONLoader
- CoNLLULoader
@ -53,7 +57,7 @@ llms:
# - AzureOpenAI
# - AzureChatOpenAI
- ChatOpenAI
- LlamaCpp
- LlamaCpp
- CTransformers
- Cohere
- Anthropic
@ -69,7 +73,7 @@ prompts:
- ZeroShotPrompt
textsplitters:
- CharacterTextSplitter
# - RecursiveCharacterTextSplitter
- RecursiveCharacterTextSplitter
# - LatexTextSplitter
# - PythonCodeTextSplitter
toolkits:

5
src/backend/langflow/custom/customs.py
View file

@ -2,7 +2,9 @@ from langflow.template import frontend_node
# These should always be instantiated
CUSTOM_NODES = {
"prompts": {"ZeroShotPrompt": frontend_node.prompts.ZeroShotPromptNode()},
"prompts": {
"ZeroShotPrompt": frontend_node.prompts.ZeroShotPromptNode(),
},
"tools": {
"PythonFunctionTool": frontend_node.tools.PythonFunctionToolNode(),
"PythonFunction": frontend_node.tools.PythonFunctionNode(),
@ -23,6 +25,7 @@ CUSTOM_NODES = {
"SeriesCharacterChain": frontend_node.chains.SeriesCharacterChainNode(),
"TimeTravelGuideChain": frontend_node.chains.TimeTravelGuideChainNode(),
"MidJourneyPromptChain": frontend_node.chains.MidJourneyPromptChainNode(),
"load_qa_chain": frontend_node.chains.CombineDocsChainNode(),
},
}

6
src/backend/langflow/graph/graph/base.py
View file

@ -10,6 +10,7 @@ from langflow.graph.vertex.types import (
)
from langflow.interface.tools.constants import FILE_TOOLS
from langflow.utils import payload
from langflow.utils.logger import logger
class Graph:
@ -24,7 +25,6 @@ class Graph:
self._edges = edges
self._build_graph()
@classmethod
@classmethod
def from_payload(cls, payload: Dict) -> "Graph":
"""
@ -43,7 +43,9 @@ class Graph:
edges = payload["edges"]
return cls(nodes, edges)
except KeyError as exc:
raise ValueError("Invalid payload") from exc
raise ValueError(
f"Invalid payload. Expected keys 'nodes' and 'edges'. Found {list(payload.keys())}"
) from exc
def _build_graph(self) -> None:
"""Builds the graph from the nodes and edges."""

5
src/backend/langflow/graph/vertex/base.py
View file

@ -49,8 +49,11 @@ class Vertex:
template_dict = self.data["node"]["template"]
self.vertex_type = (
self.data["type"] if "Tool" not in self.output else template_dict["_type"]
self.data["type"]
if "Tool" not in self.output or template_dict["_type"].islower()
else template_dict["_type"]
)
if self.base_type is None:
for base_type, value in ALL_TYPES_DICT.items():
if self.vertex_type in value:

20
src/backend/langflow/interface/agents/custom.py
View file

@ -1,4 +1,3 @@
from abc import ABC
from typing import Any, List, Optional
from langchain import LLMChain
@ -33,24 +32,7 @@ from langchain.memory.chat_memory import BaseChatMemory
from langchain.sql_database import SQLDatabase
from langchain.tools.python.tool import PythonAstREPLTool
from langchain.tools.sql_database.prompt import QUERY_CHECKER
class CustomAgentExecutor(AgentExecutor, ABC):
"""Custom agent executor"""
@staticmethod
def function_name():
return "CustomAgentExecutor"
@classmethod
def initialize(cls, *args, **kwargs):
pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def run(self, *args, **kwargs):
return super().run(*args, **kwargs)
from langflow.interface.base import CustomAgentExecutor
class JsonAgent(CustomAgentExecutor):

40
src/backend/langflow/interface/base.py
View file

@ -1,6 +1,7 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Type, Union
from langchain.chains.base import Chain
from langchain.agents import AgentExecutor
from pydantic import BaseModel
from langflow.template.field.base import TemplateField
@ -81,5 +82,42 @@ class LangChainTypeCreator(BaseModel, ABC):
)
signature.add_extra_fields()
signature.add_extra_base_classes()
return signature
class CustomChain(Chain, ABC):
"""Custom chain"""
@staticmethod
def function_name():
return "CustomChain"
@classmethod
def initialize(cls, *args, **kwargs):
pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def run(self, *args, **kwargs):
return super().run(*args, **kwargs)
class CustomAgentExecutor(AgentExecutor, ABC):
"""Custom chain"""
@staticmethod
def function_name():
return "CustomChain"
@classmethod
def initialize(cls, *args, **kwargs):
pass
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def run(self, *args, **kwargs):
return super().run(*args, **kwargs)

37
src/backend/langflow/interface/chains/base.py
View file

@ -1,12 +1,13 @@
from typing import Dict, List, Optional, Type
from typing import Any, Dict, List, Optional, Type
from langflow.custom.customs import get_custom_nodes
from langflow.interface.base import LangChainTypeCreator
from langflow.interface.custom_lists import chain_type_to_cls_dict
from langflow.interface.importing.utils import import_class
from langflow.settings import settings
from langflow.template.frontend_node.chains import ChainFrontendNode
from langflow.utils.logger import logger
from langflow.utils.util import build_template_from_class
from langflow.utils.util import build_template_from_class, build_template_from_method
from langchain import chains
# Assuming necessary imports for Field, Template, and FrontendNode classes
@ -18,10 +19,16 @@ class ChainCreator(LangChainTypeCreator):
def frontend_node_class(self) -> Type[ChainFrontendNode]:
return ChainFrontendNode
#! We need to find a better solution for this
from_method_nodes = {"ConversationalRetrievalChain": "from_llm"}
@property
def type_to_loader_dict(self) -> Dict:
if self.type_dict is None:
self.type_dict = chain_type_to_cls_dict
self.type_dict: dict[str, Any] = {
chain_name: import_class(f"langchain.chains.{chain_name}")
for chain_name in chains.__all__
}
from langflow.interface.chains.custom import CUSTOM_CHAINS
self.type_dict.update(CUSTOM_CHAINS)
@ -37,20 +44,32 @@ class ChainCreator(LangChainTypeCreator):
try:
if name in get_custom_nodes(self.type_name).keys():
return get_custom_nodes(self.type_name)[name]
elif name in self.from_method_nodes.keys():
return build_template_from_method(
name,
type_to_cls_dict=self.type_to_loader_dict,
method_name=self.from_method_nodes[name],
add_function=True,
)
return build_template_from_class(
name, self.type_to_loader_dict, add_function=True
)
except ValueError as exc:
raise ValueError("Chain not found") from exc
raise ValueError(f"Chain {name} not found: {exc}") from exc
except AttributeError as exc:
logger.error(f"Chain {name} not loaded: {exc}")
return None
def to_list(self) -> List[str]:
custom_chains = list(get_custom_nodes("chains").keys())
default_chains = list(self.type_to_loader_dict.keys())
return default_chains + custom_chains
names = []
for _, chain in self.type_to_loader_dict.items():
chain_name = (
chain.function_name()
if hasattr(chain, "function_name")
else chain.__name__
)
names.append(chain_name)
return names
chain_creator = ChainCreator()

35
src/backend/langflow/interface/chains/custom.py
View file

@ -1,11 +1,13 @@
from typing import Dict, Optional, Type
from typing import Dict, Optional, Type, Union
from langchain.chains import ConversationChain
from langchain.memory.buffer import ConversationBufferMemory
from langchain.schema import BaseMemory
from langflow.interface.base import CustomChain
from pydantic import Field, root_validator
from langchain.chains.question_answering import load_qa_chain
from langflow.interface.utils import extract_input_variables_from_prompt
from langchain.base_language import BaseLanguageModel
DEFAULT_SUFFIX = """"
Current conversation:
@ -14,7 +16,7 @@ Human: {input}
{ai_prefix}"""
class BaseCustomChain(ConversationChain):
class BaseCustomConversationChain(ConversationChain):
"""BaseCustomChain is a chain you can use to have a conversation with a custom character."""
template: Optional[str]
@ -47,7 +49,7 @@ class BaseCustomChain(ConversationChain):
return values
class SeriesCharacterChain(BaseCustomChain):
class SeriesCharacterChain(BaseCustomConversationChain):
"""SeriesCharacterChain is a chain you can use to have a conversation with a character from a series."""
character: str
@ -66,7 +68,7 @@ Human: {input}
"""Default memory store."""
class MidJourneyPromptChain(BaseCustomChain):
class MidJourneyPromptChain(BaseCustomConversationChain):
"""MidJourneyPromptChain is a chain you can use to generate new MidJourney prompts."""
template: Optional[
@ -84,7 +86,7 @@ class MidJourneyPromptChain(BaseCustomChain):
AI:""" # noqa: E501
class TimeTravelGuideChain(BaseCustomChain):
class TimeTravelGuideChain(BaseCustomConversationChain):
template: Optional[
str
] = """I want you to act as my time travel guide. You are helpful and creative. I will provide you with the historical period or future time I want to visit and you will suggest the best events, sights, or people to experience. Provide the suggestions and any necessary information.
@ -94,7 +96,26 @@ class TimeTravelGuideChain(BaseCustomChain):
AI:""" # noqa: E501
CUSTOM_CHAINS: Dict[str, Type[ConversationChain]] = {
class CombineDocsChain(CustomChain):
"""Implementation of initialize_agent function"""
@staticmethod
def function_name():
return "load_qa_chain"
@classmethod
def initialize(cls, llm: BaseLanguageModel, chain_type: str):
return load_qa_chain(llm=llm, chain_type=chain_type)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def run(self, *args, **kwargs):
return super().run(*args, **kwargs)
CUSTOM_CHAINS: Dict[str, Type[Union[ConversationChain, CustomChain]]] = {
"CombineDocsChain": CombineDocsChain,
"SeriesCharacterChain": SeriesCharacterChain,
"MidJourneyPromptChain": MidJourneyPromptChain,
"TimeTravelGuideChain": TimeTravelGuideChain,

11
src/backend/langflow/interface/custom_lists.py
View file

@ -2,7 +2,6 @@ import inspect
from typing import Any
from langchain import (
chains,
document_loaders,
embeddings,
llms,
@ -15,6 +14,8 @@ from langchain.chat_models import AzureChatOpenAI, ChatOpenAI
from langchain.chat_models import ChatAnthropic
from langflow.interface.importing.utils import import_class
from langflow.interface.agents.custom import CUSTOM_AGENTS
from langflow.interface.chains.custom import CUSTOM_CHAINS
## LLMs
llm_type_to_cls_dict = llms.type_to_cls_dict
@ -22,11 +23,6 @@ llm_type_to_cls_dict["anthropic-chat"] = ChatAnthropic # type: ignore
llm_type_to_cls_dict["azure-chat"] = AzureChatOpenAI # type: ignore
llm_type_to_cls_dict["openai-chat"] = ChatOpenAI # type: ignore
## Chains
chain_type_to_cls_dict: dict[str, Any] = {
chain_name: import_class(f"langchain.chains.{chain_name}")
for chain_name in chains.__all__
}
## Toolkits
toolkit_type_to_loader_dict: dict[str, Any] = {
@ -73,3 +69,6 @@ documentloaders_type_to_cls_dict: dict[str, Any] = {
textsplitter_type_to_cls_dict: dict[str, Any] = dict(
inspect.getmembers(text_splitter, inspect.isclass)
)
# merge CUSTOM_AGENTS and CUSTOM_CHAINS
CUSTOM_NODES = {**CUSTOM_AGENTS, **CUSTOM_CHAINS} # type: ignore

123
src/backend/langflow/interface/document_loaders/base.py
View file

@ -1,30 +1,20 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode
from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
from langflow.utils.util import build_template_from_class
def build_file_path_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a file path template for a document loader."""
return {
"type": "file",
"required": True,
"show": True,
"name": name,
"value": "",
"suffixes": suffixes,
"fileTypes": fileTypes,
}
class DocumentLoaderCreator(LangChainTypeCreator):
type_name: str = "documentloaders"
@property
def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]:
return DocumentLoaderFrontNode
@property
def type_to_loader_dict(self) -> Dict:
return documentloaders_type_to_cls_dict
@ -32,106 +22,7 @@ class DocumentLoaderCreator(LangChainTypeCreator):
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a document loader."""
try:
signature = build_template_from_class(
name, documentloaders_type_to_cls_dict
)
file_path_templates = {
"AirbyteJSONLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": build_file_path_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": build_file_path_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_file_path_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_file_path_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_file_path_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": build_file_path_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_file_path_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": build_file_path_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
"SlackDirectoryLoader": build_file_path_template(
suffixes=[".zip"], fileTypes=["zip"]
),
}
if name in file_path_templates:
signature["template"]["file_path"] = file_path_templates[name]
elif name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
signature["template"]["web_path"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_path",
"value": "",
"display_name": "Web Page",
}
elif name in {"GitbookLoader"}:
signature["template"]["web_page"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_page",
"value": "",
"display_name": "Web Page",
}
elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}:
signature["template"]["path"] = {
"type": "str",
"required": True,
"show": True,
"name": "path",
"value": "",
"display_name": "Web Page",
}
return signature
return build_template_from_class(name, documentloaders_type_to_cls_dict)
except ValueError as exc:
raise ValueError(f"Documment Loader {name} not found") from exc
except AttributeError as exc:

34
src/backend/langflow/interface/loading.py
View file

@ -19,9 +19,10 @@ from langchain.chains.loading import load_chain_from_config
from langchain.llms.loading import load_llm_from_config
from pydantic import ValidationError
from langflow.interface.agents.custom import CUSTOM_AGENTS
from langflow.interface.custom_lists import CUSTOM_NODES
from langflow.interface.importing.utils import get_function, import_by_type
from langflow.interface.toolkits.base import toolkits_creator
from langflow.interface.chains.base import chain_creator
from langflow.interface.types import get_type_list
from langflow.interface.utils import load_file_into_dict
from langflow.utils import util, validate
@ -31,10 +32,11 @@ def instantiate_class(node_type: str, base_type: str, params: Dict) -> Any:
"""Instantiate class from module type and key, and params"""
params = convert_params_to_sets(params)
params = convert_kwargs(params)
if node_type in CUSTOM_AGENTS:
custom_agent = CUSTOM_AGENTS.get(node_type)
if custom_agent:
return custom_agent.initialize(**params)
if node_type in CUSTOM_NODES:
if custom_node := CUSTOM_NODES.get(node_type):
if hasattr(custom_node, "initialize"):
return custom_node.initialize(**params)
return custom_node(**params)
class_object = import_by_type(_type=base_type, name=node_type)
return instantiate_based_on_type(class_object, base_type, node_type, params)
@ -78,10 +80,24 @@ def instantiate_based_on_type(class_object, base_type, node_type, params):
return instantiate_textsplitter(class_object, params)
elif base_type == "utilities":
return instantiate_utility(node_type, class_object, params)
elif base_type == "chains":
return instantiate_chains(node_type, class_object, params)
else:
return class_object(**params)
def instantiate_chains(node_type, class_object, params):
if "retriever" in params and hasattr(params["retriever"], "as_retriever"):
params["retriever"] = params["retriever"].as_retriever()
if node_type in chain_creator.from_method_nodes:
method = chain_creator.from_method_nodes[node_type]
if class_method := getattr(class_object, method, None):
return class_method(**params)
raise ValueError(f"Method {method} not found in {class_object}")
return class_object(**params)
def instantiate_agent(class_object, params):
return load_agent_executor(class_object, params)
@ -142,6 +158,14 @@ def instantiate_vectorstore(class_object, params):
"The source you provided did not load correctly or was empty."
"This may cause an error in the vectorstore."
)
# Chroma requires all metadata values to not be None
if class_object.__name__ == "Chroma":
for doc in params["documents"]:
if doc.metadata is None:
doc.metadata = {}
for key, value in doc.metadata.items():
if value is None:
doc.metadata[key] = ""
return class_object.from_documents(**params)

4
src/backend/langflow/interface/prompts/custom.py
View file

@ -71,7 +71,3 @@ Human: {input}
CUSTOM_PROMPTS: Dict[str, Type[BaseCustomPrompt]] = {
"SeriesCharacterPrompt": SeriesCharacterPrompt
}
if __name__ == "__main__":
prompt = SeriesCharacterPrompt(character="Harry Potter", series="Harry Potter")
print(prompt.template)

45
src/backend/langflow/interface/text_splitters/base.py
View file

@ -1,6 +1,7 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.frontend_node.textsplitters import TextSplittersFrontendNode
from langflow.interface.custom_lists import textsplitter_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
@ -10,6 +11,10 @@ from langflow.utils.util import build_template_from_class
class TextSplitterCreator(LangChainTypeCreator):
type_name: str = "textsplitters"
@property
def frontend_node_class(self) -> Type[TextSplittersFrontendNode]:
return TextSplittersFrontendNode
@property
def type_to_loader_dict(self) -> Dict:
return textsplitter_type_to_cls_dict
@ -17,43 +22,7 @@ class TextSplitterCreator(LangChainTypeCreator):
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a text splitter."""
try:
signature = build_template_from_class(name, textsplitter_type_to_cls_dict)
signature["template"]["documents"] = {
"type": "BaseLoader",
"required": True,
"show": True,
"name": "documents",
}
signature["template"]["separator"] = {
"type": "str",
"required": True,
"show": True,
"value": ".",
"name": "separator",
"display_name": "Separator",
}
signature["template"]["chunk_size"] = {
"type": "int",
"required": True,
"show": True,
"value": 1000,
"name": "chunk_size",
"display_name": "Chunk Size",
}
signature["template"]["chunk_overlap"] = {
"type": "int",
"required": True,
"show": True,
"value": 200,
"name": "chunk_overlap",
"display_name": "Chunk Overlap",
}
return signature
return build_template_from_class(name, textsplitter_type_to_cls_dict)
except ValueError as exc:
raise ValueError(f"Text Splitter {name} not found") from exc
except AttributeError as exc:

4
src/backend/langflow/template/frontend_node/__init__.py
View file

@ -7,6 +7,8 @@ from langflow.template.frontend_node import (
prompts,
tools,
vectorstores,
documentloaders,
textsplitters,
)
__all__ = [
@ -18,4 +20,6 @@ __all__ = [
"llms",
"prompts",
"vectorstores",
"documentloaders",
"textsplitters",
]

4
src/backend/langflow/template/frontend_node/agents.py
View file

@ -154,9 +154,9 @@ class CSVAgentNode(FrontendNode):
class InitializeAgentNode(FrontendNode):
name: str = "initialize_agent"
name: str = "AgentInitializer"
template: Template = Template(
type_name="initailize_agent",
type_name="initialize_agent",
fields=[
TemplateField(
field_type="str",

7
src/backend/langflow/template/frontend_node/base.py
View file

@ -14,6 +14,7 @@ class FrontendNode(BaseModel):
description: str
base_classes: List[str]
name: str = ""
display_name: str = ""
def to_dict(self) -> dict:
return {
@ -21,12 +22,16 @@ class FrontendNode(BaseModel):
"template": self.template.to_dict(self.format_field),
"description": self.description,
"base_classes": self.base_classes,
}
"display_name": self.display_name or self.name,
},
}
def add_extra_fields(self) -> None:
pass
def add_extra_base_classes(self) -> None:
pass
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
"""Formats a given field based on its attributes and value."""

52
src/backend/langflow/template/frontend_node/chains.py
View file

@ -2,10 +2,24 @@ from typing import Optional
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
from langflow.template.frontend_node.constants import QA_CHAIN_TYPES
from langflow.template.template.base import Template
class ChainFrontendNode(FrontendNode):
def add_extra_fields(self) -> None:
if self.template.type_name == "ConversationalRetrievalChain":
# add memory
self.template.add_field(
TemplateField(
field_type="BaseChatMemory",
required=False,
show=True,
name="memory",
advanced=False,
)
)
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
FrontendNode.format_field(field, name)
@ -155,3 +169,41 @@ class MidJourneyPromptChainNode(FrontendNode):
"ConversationChain",
"MidJourneyPromptChain",
]
class CombineDocsChainNode(FrontendNode):
name: str = "CombineDocsChain"
template: Template = Template(
type_name="load_qa_chain",
fields=[
TemplateField(
field_type="str",
required=True,
is_list=True,
show=True,
multiline=False,
options=QA_CHAIN_TYPES,
value=QA_CHAIN_TYPES[0],
name="chain_type",
advanced=False,
),
TemplateField(
field_type="BaseLanguageModel",
required=True,
show=True,
name="llm",
display_name="LLM",
advanced=False,
),
],
)
description: str = """Construct a zero shot agent from an LLM and tools."""
base_classes: list[str] = ["BaseCombineDocumentsChain", "function"]
def to_dict(self):
return super().to_dict()
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
# do nothing and don't return anything
pass

2
src/backend/langflow/template/frontend_node/constants.py
View file

@ -30,3 +30,5 @@ You are a good listener and you can talk about anything.
"""
HUMAN_PROMPT = "{input}"
QA_CHAIN_TYPES = ["stuff", "map_reduce", "map_rerank", "refine"]

79
src/backend/langflow/template/frontend_node/documentloaders.py Normal file
View file

@ -0,0 +1,79 @@
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class DocumentLoaderFrontNode(FrontendNode):
@staticmethod
def build_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> TemplateField:
"""Build a template field for a document loader."""
return TemplateField(
field_type="file",
required=True,
show=True,
name=name,
value="",
suffixes=suffixes,
fileTypes=fileTypes,
)
file_path_templates = {
"AirbyteJSONLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"CoNLLULoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
"CSVLoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
"UnstructuredEmailLoader": build_template(suffixes=[".eml"], fileTypes=["eml"]),
"EverNoteLoader": build_template(suffixes=[".xml"], fileTypes=["xml"]),
"FacebookChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"GutenbergLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
"BSHTMLLoader": build_template(suffixes=[".html"], fileTypes=["html"]),
"UnstructuredHTMLLoader": build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_template(suffixes=[".pdf"], fileTypes=["pdf"]),
"UnstructuredPowerPointLoader": build_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_template(suffixes=[".srt"], fileTypes=["srt"]),
"TelegramChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"TextLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
"UnstructuredWordDocumentLoader": build_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
def add_extra_fields(self) -> None:
name = None
if self.template.type_name in self.file_path_templates:
self.template.add_field(self.file_path_templates[self.template.type_name])
elif self.template.type_name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
name = "web_path"
elif self.template.type_name in {"GitbookLoader"}:
name = "web_page"
elif self.template.type_name in {"ReadTheDocsLoader"}:
name = "path"
if name:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name=name,
value="",
display_name="Web Page",
)
)

18
src/backend/langflow/template/frontend_node/memories.py
View file

@ -5,6 +5,20 @@ from langflow.template.frontend_node.base import FrontendNode
class MemoryFrontendNode(FrontendNode):
#! Needs testing
def add_extra_fields(self) -> None:
# add return_messages field
self.template.add_field(
TemplateField(
field_type="bool",
required=False,
show=True,
name="return_messages",
advanced=False,
value=False,
)
)
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
FrontendNode.format_field(field, name)
@ -18,3 +32,7 @@ class MemoryFrontendNode(FrontendNode):
field.value = 10
field.display_name = "Memory Size"
field.password = False
if field.name == "return_messages":
field.required = False
field.show = True
field.advanced = False

2
src/backend/langflow/template/frontend_node/prompts.py
View file

@ -74,7 +74,7 @@ class BasePromptFrontendNode(FrontendNode):
class ZeroShotPromptNode(BasePromptFrontendNode):
name: str = "ZeroShotPrompt"
template: Template = Template(
type_name="zero_shot",
type_name="ZeroShotPrompt",
fields=[
TemplateField(
field_type="str",

49
src/backend/langflow/template/frontend_node/textsplitters.py Normal file
View file

@ -0,0 +1,49 @@
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class TextSplittersFrontendNode(FrontendNode):
def add_extra_fields(self) -> None:
self.template.add_field(
TemplateField(
field_type="BaseLoader",
required=True,
show=True,
name="documents",
)
)
name = "separator"
if self.template.type_name == "CharacterTextSplitter":
name = "separator"
elif self.template.type_name == "RecursiveCharacterTextSplitter":
name = "separators"
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
value=".",
name=name,
display_name="Separator",
)
)
self.template.add_field(
TemplateField(
field_type="int",
required=True,
show=True,
value=1000,
name="chunk_size",
display_name="Chunk Size",
)
)
self.template.add_field(
TemplateField(
field_type="int",
required=True,
show=True,
value=200,
name="chunk_overlap",
display_name="Chunk Overlap",
)
)

2
src/backend/langflow/template/frontend_node/tools.py
View file

@ -108,7 +108,7 @@ class PythonFunctionToolNode(FrontendNode):
class PythonFunctionNode(FrontendNode):
name: str = "PythonFunction"
template: Template = Template(
type_name="python_function",
type_name="PythonFunction",
fields=[
TemplateField(
field_type="code",

3
src/backend/langflow/template/frontend_node/vectorstores.py
View file

@ -20,6 +20,9 @@ class VectorStoreFrontendNode(FrontendNode):
self.template.add_field(extra_field)
def add_extra_base_classes(self) -> None:
self.base_classes.append("BaseRetriever")
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
FrontendNode.format_field(field, name)

10
src/frontend/package-lock.json generated
View file

@ -53,6 +53,7 @@
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"shadcn-ui": "^0.1.3",
"short-unique-id": "^4.4.4",
"switch": "^0.0.0",
"table": "^6.8.1",
"tailwind-merge": "^1.13.0",
@ -8642,6 +8643,15 @@
"node": ">=8"
}
},
"node_modules/short-unique-id": {
"version": "4.4.4",
"resolved": "https://registry.npmjs.org/short-unique-id/-/short-unique-id-4.4.4.tgz",
"integrity": "sha512-oLF1NCmtbiTWl2SqdXZQbo5KM1b7axdp0RgQLq8qCBBLoq+o3A5wmLrNM6bZIh54/a8BJ3l69kTXuxwZ+XCYuw==",
"bin": {
"short-unique-id": "bin/short-unique-id",
"suid": "bin/short-unique-id"
}
},
"node_modules/side-channel": {
"version": "1.0.4",
"dev": true,

1
src/frontend/package.json
View file

@ -48,6 +48,7 @@
"remark-gfm": "^3.0.1",
"remark-math": "^5.1.1",
"shadcn-ui": "^0.1.3",
"short-unique-id": "^4.4.4",
"switch": "^0.0.0",
"table": "^6.8.1",
"tailwind-merge": "^1.13.0",

24
src/frontend/src/CustomNodes/GenericNode/index.tsx
View file

@ -33,6 +33,7 @@ import { NodeToolbar } from "reactflow";
import NodeToolbarComponent from "../../pages/FlowPage/components/nodeToolbarComponent";
import ShadTooltip from "../../components/ShadTooltipComponent";
import { postValidateNode } from "../../controllers/API";
export default function GenericNode({
data,
selected,
@ -64,17 +65,13 @@ export default function GenericNode({
const validateNode = useCallback(
debounce(async () => {
try {
const response = await fetch(`/validate/node/${data.id}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify(reactFlowInstance.toObject()),
});
const response = await postValidateNode(
data.id,
reactFlowInstance.toObject()
);
if (response.status === 200) {
let jsonResponse = await response.json();
let jsonResponseParsed = await JSON.parse(jsonResponse);
let jsonResponseParsed = await JSON.parse(response.data);
setValidationStatus(jsonResponseParsed);
}
} catch (error) {
@ -150,11 +147,10 @@ export default function GenericNode({
"Validating..."
) : (
<div className="max-h-96 overflow-auto">
{validationStatus.params
.split("\n")
.map((line, index) => (
<div key={index}>{line}</div>
))}
{validationStatus.params ||
""
.split("\n")
.map((line, index) => <div key={index}>{line}</div>)}
</div>
)
}

2
src/frontend/src/components/intComponent/index.tsx
View file

@ -36,7 +36,7 @@ export default function IntComponent({
if (disableCopyPaste) setDisableCopyPaste(false);
}}
onKeyDown={(event) => {
console.log(event);
// console.log(event);
if (
event.key !== "Backspace" &&
event.key !== "Enter" &&

23
src/frontend/src/contexts/tabsContext.tsx
View file

@ -12,7 +12,7 @@ import { updateIds, updateTemplate } from "../utils";
import { alertContext } from "./alertContext";
import { typesContext } from "./typesContext";
import { APITemplateType } from "../types/api";
import { v4 as uuidv4 } from "uuid";
import ShortUniqueId from "short-unique-id";
import { addEdge } from "reactflow";
import {
readFlowsFromDatabase,
@ -22,6 +22,8 @@ import {
uploadFlowsToDatabase,
} from "../controllers/API";
const uid = new ShortUniqueId({ length: 5 });
const TabsContextInitialValue: TabsContextType = {
tabId: "",
setTabId: (index: string) => {},
@ -29,7 +31,7 @@ const TabsContextInitialValue: TabsContextType = {
removeFlow: (id: string) => {},
addFlow: async (flowData?: any) => "",
updateFlow: (newFlow: FlowType) => {},
incrementNodeId: () => uuidv4(),
incrementNodeId: () => uid(),
downloadFlow: (flow: FlowType) => {},
downloadFlows: () => {},
uploadFlows: () => {},
@ -53,15 +55,17 @@ export const TabsContext = createContext<TabsContextType>(
export function TabsProvider({ children }: { children: ReactNode }) {
const { setErrorData, setNoticeData } = useContext(alertContext);
const [tabId, setTabId] = useState("");
const [flows, setFlows] = useState([]);
const [id, setId] = useState(uuidv4());
const [flows, setFlows] = useState<Array<FlowType>>([]);
const [id, setId] = useState(uid());
const { templates, reactFlowInstance } = useContext(typesContext);
const [lastCopiedSelection, setLastCopiedSelection] = useState(null);
const newNodeId = useRef(uuidv4());
const newNodeId = useRef(uid());
function incrementNodeId() {
newNodeId.current = uuidv4();
newNodeId.current = uid();
return newNodeId.current;
}
@ -205,10 +209,11 @@ export function TabsProvider({ children }: { children: ReactNode }) {
}
function hardReset() {
newNodeId.current = uuidv4();
newNodeId.current = uid();
setTabId("");
setFlows([]);
setId(uuidv4());
setId(uid());
}
/**
@ -249,7 +254,7 @@ export function TabsProvider({ children }: { children: ReactNode }) {
}
function getNodeId() {
return `dndnode_` + incrementNodeId();
return incrementNodeId();
}
/**

15
src/frontend/src/controllers/API/index.ts
View file

@ -22,18 +22,19 @@ export async function sendAll(data: sendAllProps) {
return await axios.post(`/api/v1/predict`, data);
}
/**
* Validates code by sending it to an API endpoint.
*
* @param {string} code - The code to validate.
* @returns {Promise<AxiosResponse<errorsTypeAPI>>} A promise that resolves to an AxiosResponse containing the validation results.
*/
export async function checkCode(
export async function postValidateCode(
code: string
): Promise<AxiosResponse<errorsTypeAPI>> {
return await axios.post("/api/v1/validate/code", { code });
}
export async function postValidateNode(
nodeId: string,
data: any
): Promise<AxiosResponse<string>> {
return await axios.post(`/api/v1/validate/node/${nodeId}`, { data });
}
/**
* Checks the prompt for the code block by sending it to an API endpoint.
*

21
src/frontend/src/modals/EditNodeModal/index.tsx
View file

@ -1,30 +1,21 @@
import {
ChevronDoubleLeftIcon,
ChevronDoubleRightIcon,
PencilSquareIcon,
XMarkIcon,
} from "@heroicons/react/24/outline";
import { Fragment, useContext, useEffect, useRef, useState } from "react";
import { useContext, useEffect, useRef, useState } from "react";
import { PopUpContext } from "../../contexts/popUpContext";
import { NodeDataType } from "../../types/flow";
import { classNames, limitScrollFieldsModal, nodeIcons } from "../../utils";
import { classNames, limitScrollFieldsModal } from "../../utils";
import { typesContext } from "../../contexts/typesContext";
import {
Table,
TableBody,
TableCaption,
TableCell,
TableHead,
TableHeader,
TableRow,
} from "../../components/ui/table";
import { Switch } from "../../components/ui/switch";
import ToggleShadComponent from "../../components/toggleShadComponent";
import { VariableIcon } from "@heroicons/react/24/outline";
import InputListComponent from "../../components/inputListComponent";
import TextAreaComponent from "../../components/textAreaComponent";
import InputComponent from "../../components/inputComponent";
import ToggleComponent from "../../components/toggleComponent";
import FloatComponent from "../../components/floatComponent";
import Dropdown from "../../components/dropdownComponent";
import IntComponent from "../../components/intComponent";
@ -43,6 +34,7 @@ import {
} from "../../components/ui/dialog";
import { Button } from "../../components/ui/button";
import { EDIT_DIALOG_SUBTITLE } from "../../constants";
import { Badge } from "../../components/ui/badge";
export default function EditNodeModal({ data }: { data: NodeDataType }) {
const [open, setOpen] = useState(true);
@ -95,11 +87,8 @@ export default function EditNodeModal({ data }: { data: NodeDataType }) {
<DialogContent className="lg:max-w-[700px] ">
<DialogHeader>
<DialogTitle className="flex items-center">
<span className="pr-2">Edit Node</span>
<PencilSquareIcon
className="h-6 w-6 text-gray-800 pl-1 dark:text-white"
aria-hidden="true"
/>
<span className="pr-2">{data.type}</span>
<Badge variant="secondary">{data.id}</Badge>
</DialogTitle>
<DialogDescription>
{EDIT_DIALOG_SUBTITLE}

4
src/frontend/src/modals/codeAreaModal/index.tsx
View file

@ -8,7 +8,7 @@ import "ace-builds/src-noconflict/theme-twilight";
import "ace-builds/src-noconflict/ext-language_tools";
// import "ace-builds/webpack-resolver";
import { darkContext } from "../../contexts/darkContext";
import { checkCode } from "../../controllers/API";
import { postValidateCode } from "../../controllers/API";
import { alertContext } from "../../contexts/alertContext";
import { TabsContext } from "../../contexts/tabsContext";
import {
@ -81,7 +81,7 @@ export default function CodeAreaModal({
<Button
className="mt-3"
onClick={() => {
checkCode(code)
postValidateCode(code)
.then((apiReturn) => {
if (apiReturn.data) {
let importsErrors = apiReturn.data.imports.errors;

4
src/frontend/src/pages/FlowPage/components/nodeToolbarComponent/index.tsx
View file

@ -61,7 +61,7 @@ const NodeToolbarComponent = (props) => {
)}
onClick={(event) => {
event.preventDefault();
console.log(reactFlowInstance.getNode(props.data.id));
// console.log(reactFlowInstance.getNode(props.data.id));
paste(
{
nodes: [reactFlowInstance.getNode(props.data.id)],
@ -94,7 +94,7 @@ const NodeToolbarComponent = (props) => {
</ShadTooltip>
)}
{/*
{/*
<Menu as="div" className="relative inline-block text-left z-100">
<button className="hover:dark:hover:bg-[#242f47] text-gray-700 transition-all duration-500 ease-in-out dark:bg-gray-800 dark:text-gray-300 shadow-md relative -ml-px inline-flex items-center bg-white px-2 py-2 ring-1 ring-inset ring-gray-300 hover:bg-muted focus:z-10 rounded-r-md">
<div>

18
src/frontend/src/pages/FlowPage/index.tsx
View file

@ -3,17 +3,19 @@ import Page from "./components/PageComponent";
import { TabsContext } from "../../contexts/tabsContext";
import { useParams } from "react-router-dom";
export default function FlowPage(){
export default function FlowPage() {
const { flows, tabId, setTabId } = useContext(TabsContext);
const {id} = useParams();
const { id } = useParams();
useEffect(() => {
setTabId(id);
}, [id])
}, [id]);
return (
<div className="h-full w-full overflow-hidden">
{flows.length > 0 && tabId !== "" && flows.findIndex(flow => flow.id === tabId) !== -1 &&
<Page flow={flows.find(flow => flow.id === tabId)} />
}
{flows.length > 0 &&
tabId !== "" &&
flows.findIndex((flow) => flow.id === tabId) !== -1 && (
<Page flow={flows.find((flow) => flow.id === tabId)} />
)}
</div>
)
}
);
}

2
tests/data/complex_example.json
View file

@ -352,7 +352,7 @@
"type": "str",
"list": false
},
"_type": "zero_shot"
"_type": "ZeroShotPrompt"
},
"description": "Prompt template for Zero Shot Agent.",
"base_classes": [

13
tests/test_agents_template.py
View file

@ -1,15 +1,4 @@
from fastapi.testclient import TestClient
from langflow.settings import settings
# check that all agents are in settings.agents
# are in json_response["agents"]
def test_agents_settings(client: TestClient):
response = client.get("api/v1/all")
assert response.status_code == 200
json_response = response.json()
agents = json_response["agents"]
assert set(agents.keys()) == set(settings.agents)
def test_zero_shot_agent(client: TestClient):
@ -131,7 +120,7 @@ def test_initialize_agent(client: TestClient):
json_response = response.json()
agents = json_response["agents"]
initialize_agent = agents["initialize_agent"]
initialize_agent = agents["AgentInitializer"]
assert initialize_agent["base_classes"] == ["AgentExecutor", "function"]
template = initialize_agent["template"]