jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge branch 'dev' into agent_and_qa_chain

Browse source
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-06-09 16:25:29 -03:00 • committed by GitHub
commit fdf24f269b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 5290 additions and 2640 deletions
Download patch file Download diff file Expand all files Collapse all files

105
src/backend/langflow/__main__.py
View file

@ -1,20 +1,28 @@
import multiprocessing
import sys
import time
import httpx
from multiprocess import Process, cpu_count # type: ignore
import platform
from pathlib import Path
from rich.panel import Panel
from rich import box
from rich import print as rprint
import typer
from fastapi.staticfiles import StaticFiles
from langflow.main import create_app
from langflow.settings import settings
from langflow.utils.logger import configure
from langflow.utils.logger import configure, logger
import webbrowser
app = typer.Typer()
def get_number_of_workers(workers=None):
if workers == -1:
workers = (multiprocessing.cpu_count() * 2) + 1
workers = (cpu_count() * 2) + 1
return workers
@ -77,10 +85,17 @@ def serve(
timeout: int = typer.Option(60, help="Worker timeout in seconds."),
port: int = typer.Option(7860, help="Port to listen on."),
config: str = typer.Option("config.yaml", help="Path to the configuration file."),
log_level: str = typer.Option("info", help="Logging level."),
log_level: str = typer.Option("critical", help="Logging level."),
log_file: Path = typer.Option("logs/langflow.log", help="Path to the log file."),
jcloud: bool = typer.Option(False, help="Deploy on Jina AI Cloud"),
dev: bool = typer.Option(False, help="Run in development mode (may contain bugs)"),
path: str = typer.Option(
None,
help="Path to the frontend directory containing build files. This is for development purposes only.",
),
open_browser: bool = typer.Option(
True, help="Open the browser after starting the server."
),
):
"""
Run the Langflow server.
@ -93,8 +108,11 @@ def serve(
update_settings(config, dev=dev)
app = create_app()
# get the directory of the current file
path = Path(__file__).parent
static_files_dir = path / "frontend"
if not path:
frontend_path = Path(__file__).parent
static_files_dir = frontend_path / "frontend"
else:
static_files_dir = Path(path)
app.mount(
"/",
StaticFiles(directory=static_files_dir, html=True),
@ -107,17 +125,74 @@ def serve(
"timeout": timeout,
}
if platform.system() in ["Darwin", "Windows"]:
# Run using uvicorn on MacOS and Windows
# Windows doesn't support gunicorn
# MacOS requires an env variable to be set to use gunicorn
import uvicorn
webapp_process = Process(
target=run_langflow, args=(host, port, log_level, options, app)
)
webapp_process.start()
status_code = 0
while status_code != 200:
try:
status_code = httpx.get(f"http://{host}:{port}").status_code
except Exception:
time.sleep(1)
uvicorn.run(app, host=host, port=port, log_level=log_level)
else:
from langflow.server import LangflowApplication
print_banner(host, port)
if open_browser:
webbrowser.open(f"http://{host}:{port}")
LangflowApplication(app, options).run()
def print_banner(host, port):
# console = Console()
word = "LangFlow"
colors = ["#690080", "#660099", "#4d00b3", "#3300cc", "#1a00e6", "#0000ff"]
styled_word = ""
for i, char in enumerate(word):
color = colors[i % len(colors)]
styled_word += f"[{color}]{char}[/]"
# Title with emojis and gradient text
title = (
f"[bold]Welcome to :chains: {styled_word} [/bold]\n\n"
f"Access [link=http://{host}:{port}]http://{host}:{port}[/link]"
)
info_text = (
"Collaborate, and contribute at our "
"[bold][link=https://github.com/logspace-ai/langflow]GitHub Repo[/link][/bold] :rocket:"
)
# Create a panel with the title and the info text, and a border around it
panel = Panel(
f"{title}\n{info_text}", box=box.ROUNDED, border_style="blue", expand=False
)
# Print the banner with a separator line before and after
rprint(panel)
def run_langflow(host, port, log_level, options, app):
"""
Run Langflow server on localhost
"""
try:
if platform.system() in ["Darwin", "Windows"]:
# Run using uvicorn on MacOS and Windows
# Windows doesn't support gunicorn
# MacOS requires an env variable to be set to use gunicorn
import uvicorn
uvicorn.run(app, host=host, port=port, log_level=log_level)
else:
from langflow.server import LangflowApplication
LangflowApplication(app, options).run()
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(e)
sys.exit(1)
def main():

4
src/backend/langflow/config.yaml
View file

@ -57,7 +57,7 @@ llms:
# - AzureOpenAI
# - AzureChatOpenAI
- ChatOpenAI
- LlamaCpp
- LlamaCpp
- CTransformers
- Cohere
- Anthropic
@ -73,7 +73,7 @@ prompts:
- ZeroShotPrompt
textsplitters:
- CharacterTextSplitter
# - RecursiveCharacterTextSplitter
- RecursiveCharacterTextSplitter
# - LatexTextSplitter
# - PythonCodeTextSplitter
toolkits:

123
src/backend/langflow/interface/document_loaders/base.py
View file

@ -1,30 +1,20 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.frontend_node.documentloaders import DocumentLoaderFrontNode
from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
from langflow.utils.util import build_template_from_class
def build_file_path_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> Dict:
"""Build a file path template for a document loader."""
return {
"type": "file",
"required": True,
"show": True,
"name": name,
"value": "",
"suffixes": suffixes,
"fileTypes": fileTypes,
}
class DocumentLoaderCreator(LangChainTypeCreator):
type_name: str = "documentloaders"
@property
def frontend_node_class(self) -> Type[DocumentLoaderFrontNode]:
return DocumentLoaderFrontNode
@property
def type_to_loader_dict(self) -> Dict:
return documentloaders_type_to_cls_dict
@ -32,106 +22,7 @@ class DocumentLoaderCreator(LangChainTypeCreator):
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a document loader."""
try:
signature = build_template_from_class(
name, documentloaders_type_to_cls_dict
)
file_path_templates = {
"AirbyteJSONLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"CoNLLULoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"CSVLoader": build_file_path_template(
suffixes=[".csv"], fileTypes=["csv"]
),
"UnstructuredEmailLoader": build_file_path_template(
suffixes=[".eml"], fileTypes=["eml"]
),
"EverNoteLoader": build_file_path_template(
suffixes=[".xml"], fileTypes=["xml"]
),
"FacebookChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"GutenbergLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"BSHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredHTMLLoader": build_file_path_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_file_path_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_file_path_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_file_path_template(
suffixes=[".pdf"], fileTypes=["pdf"]
),
"UnstructuredPowerPointLoader": build_file_path_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_file_path_template(
suffixes=[".srt"], fileTypes=["srt"]
),
"TelegramChatLoader": build_file_path_template(
suffixes=[".json"], fileTypes=["json"]
),
"TextLoader": build_file_path_template(
suffixes=[".txt"], fileTypes=["txt"]
),
"UnstructuredWordDocumentLoader": build_file_path_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
"SlackDirectoryLoader": build_file_path_template(
suffixes=[".zip"], fileTypes=["zip"]
),
}
if name in file_path_templates:
signature["template"]["file_path"] = file_path_templates[name]
elif name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
signature["template"]["web_path"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_path",
"value": "",
"display_name": "Web Page",
}
elif name in {"GitbookLoader"}:
signature["template"]["web_page"] = {
"type": "str",
"required": True,
"show": True,
"name": "web_page",
"value": "",
"display_name": "Web Page",
}
elif name in {"ReadTheDocsLoader", "NotionDirectoryLoader"}:
signature["template"]["path"] = {
"type": "str",
"required": True,
"show": True,
"name": "path",
"value": "",
"display_name": "Web Page",
}
return signature
return build_template_from_class(name, documentloaders_type_to_cls_dict)
except ValueError as exc:
raise ValueError(f"Documment Loader {name} not found") from exc
except AttributeError as exc:

45
src/backend/langflow/interface/text_splitters/base.py
View file

@ -1,6 +1,7 @@
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Type
from langflow.interface.base import LangChainTypeCreator
from langflow.template.frontend_node.textsplitters import TextSplittersFrontendNode
from langflow.interface.custom_lists import textsplitter_type_to_cls_dict
from langflow.settings import settings
from langflow.utils.logger import logger
@ -10,6 +11,10 @@ from langflow.utils.util import build_template_from_class
class TextSplitterCreator(LangChainTypeCreator):
type_name: str = "textsplitters"
@property
def frontend_node_class(self) -> Type[TextSplittersFrontendNode]:
return TextSplittersFrontendNode
@property
def type_to_loader_dict(self) -> Dict:
return textsplitter_type_to_cls_dict
@ -17,43 +22,7 @@ class TextSplitterCreator(LangChainTypeCreator):
def get_signature(self, name: str) -> Optional[Dict]:
"""Get the signature of a text splitter."""
try:
signature = build_template_from_class(name, textsplitter_type_to_cls_dict)
signature["template"]["documents"] = {
"type": "BaseLoader",
"required": True,
"show": True,
"name": "documents",
}
signature["template"]["separator"] = {
"type": "str",
"required": True,
"show": True,
"value": ".",
"name": "separator",
"display_name": "Separator",
}
signature["template"]["chunk_size"] = {
"type": "int",
"required": True,
"show": True,
"value": 1000,
"name": "chunk_size",
"display_name": "Chunk Size",
}
signature["template"]["chunk_overlap"] = {
"type": "int",
"required": True,
"show": True,
"value": 200,
"name": "chunk_overlap",
"display_name": "Chunk Overlap",
}
return signature
return build_template_from_class(name, textsplitter_type_to_cls_dict)
except ValueError as exc:
raise ValueError(f"Text Splitter {name} not found") from exc
except AttributeError as exc:

4
src/backend/langflow/interface/utils.py
View file

@ -47,9 +47,9 @@ def try_setting_streaming_options(langchain_object, websocket):
llm = langchain_object.llm_chain.llm
if isinstance(llm, BaseLanguageModel):
if hasattr(llm, "streaming"):
if hasattr(llm, "streaming") and isinstance(llm.streaming, bool):
llm.streaming = True
if hasattr(llm, "stream"):
elif hasattr(llm, "stream") and isinstance(llm.stream, bool):
llm.stream = True
return langchain_object

4
src/backend/langflow/template/frontend_node/__init__.py
View file

@ -7,6 +7,8 @@ from langflow.template.frontend_node import (
prompts,
tools,
vectorstores,
documentloaders,
textsplitters,
)
__all__ = [
@ -18,4 +20,6 @@ __all__ = [
"llms",
"prompts",
"vectorstores",
"documentloaders",
"textsplitters",
]

79
src/backend/langflow/template/frontend_node/documentloaders.py Normal file
View file

@ -0,0 +1,79 @@
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class DocumentLoaderFrontNode(FrontendNode):
@staticmethod
def build_template(
suffixes: list, fileTypes: list, name: str = "file_path"
) -> TemplateField:
"""Build a template field for a document loader."""
return TemplateField(
field_type="file",
required=True,
show=True,
name=name,
value="",
suffixes=suffixes,
fileTypes=fileTypes,
)
file_path_templates = {
"AirbyteJSONLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"CoNLLULoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
"CSVLoader": build_template(suffixes=[".csv"], fileTypes=["csv"]),
"UnstructuredEmailLoader": build_template(suffixes=[".eml"], fileTypes=["eml"]),
"EverNoteLoader": build_template(suffixes=[".xml"], fileTypes=["xml"]),
"FacebookChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"GutenbergLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
"BSHTMLLoader": build_template(suffixes=[".html"], fileTypes=["html"]),
"UnstructuredHTMLLoader": build_template(
suffixes=[".html"], fileTypes=["html"]
),
"UnstructuredImageLoader": build_template(
suffixes=[".jpg", ".jpeg", ".png", ".gif", ".bmp"],
fileTypes=["jpg", "jpeg", "png", "gif", "bmp"],
),
"UnstructuredMarkdownLoader": build_template(
suffixes=[".md"], fileTypes=["md"]
),
"PyPDFLoader": build_template(suffixes=[".pdf"], fileTypes=["pdf"]),
"UnstructuredPowerPointLoader": build_template(
suffixes=[".pptx", ".ppt"], fileTypes=["pptx", "ppt"]
),
"SRTLoader": build_template(suffixes=[".srt"], fileTypes=["srt"]),
"TelegramChatLoader": build_template(suffixes=[".json"], fileTypes=["json"]),
"TextLoader": build_template(suffixes=[".txt"], fileTypes=["txt"]),
"UnstructuredWordDocumentLoader": build_template(
suffixes=[".docx", ".doc"], fileTypes=["docx", "doc"]
),
}
def add_extra_fields(self) -> None:
name = None
if self.template.type_name in self.file_path_templates:
self.template.add_field(self.file_path_templates[self.template.type_name])
elif self.template.type_name in {
"WebBaseLoader",
"AZLyricsLoader",
"CollegeConfidentialLoader",
"HNLoader",
"IFixitLoader",
"IMSDbLoader",
}:
name = "web_path"
elif self.template.type_name in {"GitbookLoader"}:
name = "web_page"
elif self.template.type_name in {"ReadTheDocsLoader"}:
name = "path"
if name:
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name=name,
value="",
display_name="Web Page",
)
)

18
src/backend/langflow/template/frontend_node/llms.py
View file

@ -16,20 +16,30 @@ class LLMFrontendNode(FrontendNode):
def format_azure_field(field: TemplateField):
if field.name == "model_name":
field.show = False # Azure uses deployment_name instead of model_name.
if field.name == "openai_api_type":
elif field.name == "openai_api_type":
field.show = False
field.password = False
field.value = "azure"
if field.name == "openai_api_version":
elif field.name == "openai_api_version":
field.password = False
field.value = "2023-03-15-preview"
@staticmethod
def format_llama_field(field: TemplateField):
field.show = True
field.advanced = not field.required
@staticmethod
def format_field(field: TemplateField, name: Optional[str] = None) -> None:
display_names_dict = {
"huggingfacehub_api_token": "HuggingFace Hub API Token",
}
FrontendNode.format_field(field, name)
LLMFrontendNode.format_openai_field(field)
if name and "azure" in name.lower():
LLMFrontendNode.format_azure_field(field)
if name and "llama" in name.lower():
LLMFrontendNode.format_llama_field(field)
SHOW_FIELDS = ["repo_id"]
if field.name in SHOW_FIELDS:
field.show = True
@ -65,7 +75,3 @@ class LLMFrontendNode(FrontendNode):
]:
field.advanced = False
field.show = True
LLMFrontendNode.format_openai_field(field)
if name and "azure" in name.lower():
LLMFrontendNode.format_azure_field(field)

49
src/backend/langflow/template/frontend_node/textsplitters.py Normal file
View file

@ -0,0 +1,49 @@
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
class TextSplittersFrontendNode(FrontendNode):
def add_extra_fields(self) -> None:
self.template.add_field(
TemplateField(
field_type="BaseLoader",
required=True,
show=True,
name="documents",
)
)
name = "separator"
if self.template.type_name == "CharacterTextSplitter":
name = "separator"
elif self.template.type_name == "RecursiveCharacterTextSplitter":
name = "separators"
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
value=".",
name=name,
display_name="Separator",
)
)
self.template.add_field(
TemplateField(
field_type="int",
required=True,
show=True,
value=1000,
name="chunk_size",
display_name="Chunk Size",
)
)
self.template.add_field(
TemplateField(
field_type="int",
required=True,
show=True,
value=200,
name="chunk_overlap",
display_name="Chunk Overlap",
)
)