jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Refactor code and update dependencies (#1241)

Browse source 
This pull request includes multiple code refactorings and updates to dependencies. It also removes unused imports, fixes code formatting, and updates the langflow version to 0.6.3a6.
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-12-22 11:22:37 -03:00 • committed by GitHub
commit afd1bc0c18
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 826 additions and 651 deletions
Download patch file Download diff file Expand all files Collapse all files

1005
poetry.lock generated
View file

File diff suppressed because it is too large Load diff

2
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "langflow"
version = "0.6.3a5"
version = "0.6.3a6"
description = "A Python package with a built-in web application"
authors = ["Logspace <contact@logspace.ai>"]
maintainers = [

12
src/backend/langflow/api/v1/chat.py
View file

@ -1,19 +1,15 @@
import time
from fastapi import (APIRouter, Depends, HTTPException, Query, WebSocket,
WebSocketException, status)
from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketException, status
from fastapi.responses import StreamingResponse
from langflow.api.utils import build_input_keys_response, format_elapsed_time
from langflow.api.v1.schemas import (BuildStatus, BuiltResponse, InitResponse,
StreamData)
from langflow.api.v1.schemas import BuildStatus, BuiltResponse, InitResponse, StreamData
from langflow.graph.graph.base import Graph
from langflow.services.auth.utils import (get_current_active_user,
get_current_user_by_jwt)
from langflow.services.auth.utils import get_current_active_user, get_current_user_by_jwt
from langflow.services.cache.service import BaseCacheService
from langflow.services.cache.utils import update_build_status
from langflow.services.chat.service import ChatService
from langflow.services.deps import (get_cache_service, get_chat_service,
get_session)
from langflow.services.deps import get_cache_service, get_chat_service, get_session
from loguru import logger
from sqlmodel import Session

11
src/backend/langflow/api/v1/endpoints.py
View file

@ -1,5 +1,5 @@
from http import HTTPStatus
from typing import Annotated, List, Optional, Union
from typing import Annotated, Any, List, Optional, Union
import sqlalchemy as sa
from fastapi import APIRouter, Body, Depends, HTTPException, UploadFile, status
@ -16,7 +16,7 @@ from langflow.api.v1.schemas import (
)
from langflow.interface.custom.custom_component import CustomComponent
from langflow.interface.custom.directory_reader import DirectoryReader
from langflow.interface.custom.utils import build_custom_component_template, create_and_validate_component
from langflow.interface.custom.utils import build_custom_component_template
from langflow.processing.process import process_graph_cached, process_tweaks
from langflow.services.auth.utils import api_key_security, get_current_active_user
from langflow.services.cache.utils import save_uploaded_file
@ -49,7 +49,7 @@ async def process_graph_data(
task_service: "TaskService" = Depends(get_task_service),
sync: bool = True,
):
task_result = None
task_result: Any = None
task_status = None
if tweaks:
try:
@ -270,7 +270,7 @@ async def custom_component(
raw_code: CustomComponentCode,
user: User = Depends(get_current_active_user),
):
component = create_and_validate_component(raw_code.code)
component = CustomComponent(code=raw_code.code)
built_frontend_node = build_custom_component_template(component, user_id=user.id)
@ -289,7 +289,6 @@ async def reload_custom_component(path: str, user: User = Depends(get_current_ac
raise ValueError(content)
extractor = CustomComponent(code=content)
extractor.validate()
return build_custom_component_template(extractor, user_id=user.id)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc))
@ -300,7 +299,7 @@ async def custom_component_update(
raw_code: CustomComponentCode,
user: User = Depends(get_current_active_user),
):
component = create_and_validate_component(raw_code.code)
component = CustomComponent(code=raw_code.code)
component_node = build_custom_component_template(component, user_id=user.id, update_field=raw_code.field)
# Update the field

39
src/backend/langflow/components/agents/AgentInitializer.py
View file

@ -1,7 +1,6 @@
from typing import Callable, List, Union
from typing import Callable, List, Optional, Union
from langchain.agents import AgentExecutor, AgentType, initialize_agent, types
from langflow import CustomComponent
from langflow.field_typing import BaseChatMemory, BaseLanguageModel, Tool
@ -20,18 +19,34 @@ class AgentInitializerComponent(CustomComponent):
"memory": {"display_name": "Memory"},
"tools": {"display_name": "Tools"},
"llm": {"display_name": "Language Model"},
"code": {"advanced": True},
}
def build(
self, agent: str, llm: BaseLanguageModel, memory: BaseChatMemory, tools: List[Tool], max_iterations: int
self,
agent: str,
llm: BaseLanguageModel,
tools: List[Tool],
max_iterations: int,
memory: Optional[BaseChatMemory] = None,
) -> Union[AgentExecutor, Callable]:
agent = AgentType(agent)
return initialize_agent(
tools=tools,
llm=llm,
agent=agent,
memory=memory,
return_intermediate_steps=True,
handle_parsing_errors=True,
max_iterations=max_iterations,
)
if memory:
return initialize_agent(
tools=tools,
llm=llm,
agent=agent,
memory=memory,
return_intermediate_steps=True,
handle_parsing_errors=True,
max_iterations=max_iterations,
)
else:
return initialize_agent(
tools=tools,
llm=llm,
agent=agent,
return_intermediate_steps=True,
handle_parsing_errors=True,
max_iterations=max_iterations,
)

17
src/backend/langflow/components/llms/AzureChatOpenAI.py
View file

@ -3,6 +3,7 @@ from langflow import CustomComponent
from langchain.llms.base import BaseLanguageModel
from langchain.chat_models.azure_openai import AzureChatOpenAI
class AzureChatOpenAIComponent(CustomComponent):
display_name: str = "AzureChatOpenAI"
description: str = "LLM model from Azure OpenAI."
@ -16,7 +17,7 @@ class AzureChatOpenAIComponent(CustomComponent):
"gpt-4-32k",
"gpt-4-vision",
]
def build_config(self):
return {
"model": {
@ -28,7 +29,7 @@ class AzureChatOpenAIComponent(CustomComponent):
"azure_endpoint": {
"display_name": "Azure Endpoint",
"required": True,
"info": "Your Azure endpoint, including the resource.. Example: `https://example-resource.azure.openai.com/`"
"info": "Your Azure endpoint, including the resource.. Example: `https://example-resource.azure.openai.com/`",
},
"azure_deployment": {
"display_name": "Deployment Name",
@ -40,18 +41,14 @@ class AzureChatOpenAIComponent(CustomComponent):
"required": True,
"advanced": True,
},
"api_key": {
"display_name": "API Key",
"required": True,
"password": True
},
"api_key": {"display_name": "API Key", "required": True, "password": True},
"temperature": {
"display_name": "Temperature",
"value": 0.7,
"field_type": "float",
"required": False,
},
"max_tokens": {
"max_tokens": {
"display_name": "Max Tokens",
"value": 1000,
"required": False,
@ -71,8 +68,6 @@ class AzureChatOpenAIComponent(CustomComponent):
temperature: float = 0.7,
max_tokens: Optional[int] = 1000,
) -> BaseLanguageModel:
return AzureChatOpenAI(
model=model,
azure_endpoint=azure_endpoint,
@ -80,5 +75,5 @@ class AzureChatOpenAIComponent(CustomComponent):
api_version=api_version,
api_key=api_key,
temperature=temperature,
max_tokens=max_tokens
max_tokens=max_tokens,
)

27
src/backend/langflow/components/vectorstores/Weaviate.py
View file

@ -1,4 +1,4 @@
import weaviate # type: ignore
import weaviate # type: ignore
from typing import Optional, Union
from langflow import CustomComponent
@ -12,18 +12,29 @@ from langchain.embeddings.base import Embeddings
class WeaviateVectorStore(CustomComponent):
display_name: str = "Weaviate"
description: str = "Implementation of Vector Store using Weaviate"
documentation = (
"https://python.langchain.com/docs/integrations/vectorstores/weaviate"
)
documentation = "https://python.langchain.com/docs/integrations/vectorstores/weaviate"
beta = True
field_config = {
"url": {"display_name": "Weaviate URL", "value": "http://localhost:8080"},
"api_key": { "display_name": "API Key", "password": True,"required": False, },
"index_name": {"display_name": "Index name","required": False,},
"text_key": {"display_name": "Text Key","required": False, "advanced": True, "value": "text"},
"api_key": {
"display_name": "API Key",
"password": True,
"required": False,
},
"index_name": {
"display_name": "Index name",
"required": False,
},
"text_key": {"display_name": "Text Key", "required": False, "advanced": True, "value": "text"},
"documents": {"display_name": "Documents", "is_list": True},
"embedding": {"display_name": "Embedding"},
"attributes": {"display_name": "Attributes", "required": False, "is_list": True, "field_type": "str", "advanced": True},
"attributes": {
"display_name": "Attributes",
"required": False,
"is_list": True,
"field_type": "str",
"advanced": True,
},
"search_by_text": {"display_name": "Search By Text", "field_type": "bool", "advanced": True},
"code": {"show": False},
}

5
src/backend/langflow/interface/agents/custom.py
View file

@ -1,6 +1,6 @@
from typing import Any, List, Optional
from typing import Any, Optional
from langchain.agents import AgentExecutor, AgentType, Tool, ZeroShotAgent, initialize_agent
from langchain.agents import AgentExecutor, ZeroShotAgent
from langchain.agents.agent_toolkits import (
SQLDatabaseToolkit,
VectorStoreInfo,
@ -15,7 +15,6 @@ from langchain.agents.agent_toolkits.vectorstore.prompt import ROUTER_PREFIX as
from langchain.agents.mrkl.prompt import FORMAT_INSTRUCTIONS
from langchain.base_language import BaseLanguageModel
from langchain.chains.llm import LLMChain
from langchain.memory.chat_memory import BaseChatMemory
from langchain.sql_database import SQLDatabase
from langchain.tools.sql_database.prompt import QUERY_CHECKER
from langchain_experimental.agents.agent_toolkits.pandas.prompt import PREFIX as PANDAS_PREFIX

3
src/backend/langflow/interface/custom/code_parser/code_parser.py
View file

@ -6,8 +6,7 @@ from typing import Any, Dict, List, Type, Union
from cachetools import TTLCache, cachedmethod, keys
from fastapi import HTTPException
from langflow.interface.custom.schema import (CallableCodeDetails,
ClassCodeDetails)
from langflow.interface.custom.schema import CallableCodeDetails, ClassCodeDetails
class CodeSyntaxError(HTTPException):

2
src/backend/langflow/interface/custom/code_parser/utils.py
View file

@ -36,4 +36,4 @@ def extract_union_types_from_generic_alias(return_type: GenericAlias) -> list:
"""
Extracts the inner type from a type hint that is a Union.
"""
return list(return_type.__args__)
return list(return_type.__args__)

36
src/backend/langflow/interface/custom/custom_component/custom_component.py
View file

@ -5,10 +5,11 @@ from uuid import UUID
import yaml
from cachetools import TTLCache, cachedmethod
from fastapi import HTTPException
from langflow.interface.custom.code_parser.utils import (
extract_inner_type_from_generic_alias,
extract_union_types_from_generic_alias)
from langflow.interface.custom.directory_reader import DirectoryReader
extract_union_types_from_generic_alias,
)
from langflow.services.database.models.flow import Flow
from langflow.services.database.utils import session_getter
from langflow.services.deps import get_credential_service, get_db_service
@ -46,34 +47,6 @@ class CustomComponent(Component):
def build_config(self):
return self.field_config
def _class_template_validation(self, code: str):
TYPE_HINT_LIST = ["Optional", "Prompt", "PromptTemplate", "LLMChain"]
if not code:
raise HTTPException(
status_code=400,
detail={
"error": self.ERROR_CODE_NULL,
"traceback": "",
},
)
reader = DirectoryReader("", False)
for type_hint in TYPE_HINT_LIST:
if reader._is_type_hint_used_in_args(type_hint, code) and not reader._is_type_hint_imported(
type_hint, code
):
error_detail = {
"error": "Type hint Error",
"traceback": f"Type hint '{type_hint}' is used but not imported in the code.",
}
raise HTTPException(status_code=400, detail=error_detail)
return True
def validate(self) -> bool:
return self._class_template_validation(self.code) if self.code else False
@property
def tree(self):
return self.get_code_tree(self.code or "")
@ -206,8 +179,7 @@ class CustomComponent(Component):
return validate.create_function(self.code, self.function_entrypoint_name)
async def load_flow(self, flow_id: str, tweaks: Optional[dict] = None) -> Any:
from langflow.processing.process import (build_sorted_vertices,
process_tweaks)
from langflow.processing.process import build_sorted_vertices, process_tweaks
db_service = get_db_service()
with session_getter(db_service) as session:

3
src/backend/langflow/interface/custom/directory_reader/directory_reader.py
View file

@ -65,12 +65,13 @@ class DirectoryReader:
def filter_loaded_components(self, data: dict, with_errors: bool) -> dict:
from langflow.interface.custom.utils import build_component
items = [
{
"name": menu["name"],
"path": menu["path"],
"components": [
(*build_component(component),component)
(*build_component(component), component)
for component in menu["components"]
if (component["error"] if with_errors else not component["error"])
],

12
src/backend/langflow/interface/custom/directory_reader/utils.py
View file

@ -1,6 +1,5 @@
from langflow.interface.custom.directory_reader import DirectoryReader
from langflow.template.frontend_node.custom_components import \
CustomComponentFrontendNode
from langflow.template.frontend_node.custom_components import CustomComponentFrontendNode
from loguru import logger
@ -75,10 +74,9 @@ def create_invalid_component_template(component, component_name):
"""Create a template for an invalid component."""
component_code = component["code"]
component_frontend_node = CustomComponentFrontendNode(
description="ERROR - Check your Python Code",
display_name=f"ERROR - {component_name}",
)
description="ERROR - Check your Python Code",
display_name=f"ERROR - {component_name}",
)
component_frontend_node.error = component.get("error", None)
field = component_frontend_node.template.get_field("code")
@ -144,4 +142,4 @@ def build_menu_items(menu_item):
except Exception as exc:
logger.error(f"Error loading Component: {component['output_types']}")
logger.exception(f"Error while building custom component {component['output_types']}: {exc}")
return menu_items
return menu_items

26
src/backend/langflow/interface/custom/utils.py
View file

@ -7,18 +7,20 @@ from typing import Any, Dict, List, Optional, Union
from uuid import UUID
from fastapi import HTTPException
from loguru import logger
from langflow.field_typing.range_spec import RangeSpec
from langflow.interface.custom.code_parser.utils import extract_inner_type
from langflow.interface.custom.custom_component import CustomComponent
from langflow.interface.custom.directory_reader.utils import (
build_custom_component_list_from_path, determine_component_name,
merge_nested_dicts_with_renaming)
build_custom_component_list_from_path,
determine_component_name,
merge_nested_dicts_with_renaming,
)
from langflow.interface.importing.utils import eval_custom_component_code
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.custom_components import \
CustomComponentFrontendNode
from langflow.template.frontend_node.custom_components import CustomComponentFrontendNode
from langflow.utils.util import get_base_classes
from loguru import logger
def add_output_types(frontend_node: CustomComponentFrontendNode, return_types: List[str]):
@ -174,9 +176,7 @@ def get_field_dict(field: Union[TemplateField, dict]):
return field
def run_build_config(
custom_component: CustomComponent, user_id: Optional[Union[str, UUID]] = None, update_field=None
):
def run_build_config(custom_component: CustomComponent, user_id: Optional[Union[str, UUID]] = None, update_field=None):
"""Build the field configuration for a custom component"""
try:
@ -311,7 +311,6 @@ def create_component_template(component):
component_output_types = component["output_types"]
component_extractor = CustomComponent(code=component_code)
component_extractor.validate()
component_template = build_custom_component_template(component_extractor)
component_template["output_types"] = component_output_types
@ -343,12 +342,6 @@ def build_custom_components(settings_service):
return custom_components_from_file
def create_and_validate_component(code: str) -> CustomComponent:
component = CustomComponent(code=code)
component.validate()
return component
def update_field_dict(field_dict):
"""Update the field dictionary by calling options() or value() if they are callable"""
if "options" in field_dict and callable(field_dict["options"]):
@ -378,6 +371,3 @@ def build_component(component):
component_name = determine_component_name(component)
component_template = create_component_template(component)
return component_name, component_template

6
src/backend/langflow/interface/types.py
View file

@ -1,9 +1,7 @@
from cachetools import LRUCache, cached
from langflow.interface.agents.base import agent_creator
from langflow.interface.chains.base import chain_creator
from langflow.interface.custom.directory_reader.utils import \
merge_nested_dicts_with_renaming
from langflow.interface.custom.directory_reader.utils import merge_nested_dicts_with_renaming
from langflow.interface.custom.utils import build_custom_components
from langflow.interface.document_loaders.base import documentloader_creator
from langflow.interface.embeddings.base import embedding_creator
@ -70,5 +68,3 @@ def get_all_types_dict(settings_service):
native_components = build_langchain_types_dict()
custom_components_from_file = build_custom_components(settings_service)
return merge_nested_dicts_with_renaming(native_components, custom_components_from_file)

22
src/backend/langflow/processing/process.py
View file

@ -7,12 +7,11 @@ from langchain.schema import AgentAction, Document
from langchain.vectorstores.base import VectorStore
from langchain_core.messages import AIMessage
from langchain_core.runnables.base import Runnable
from loguru import logger
from pydantic import BaseModel
from langflow.interface.custom.custom_component import CustomComponent
from langflow.interface.run import build_sorted_vertices, get_memory_key, update_memory_keys
from langflow.services.deps import get_session_service
from loguru import logger
from pydantic import BaseModel
def fix_memory_inputs(langchain_object):
@ -107,10 +106,23 @@ def get_build_result(data_graph, session_id):
return build_sorted_vertices(data_graph)
def process_inputs(inputs: Optional[dict], artifacts: Dict[str, Any]) -> dict:
def process_inputs(
inputs: Optional[Union[dict, List[dict]]] = None, artifacts: Optional[Dict[str, Any]] = None
) -> Union[dict, List[dict]]:
if inputs is None:
inputs = {}
if artifacts is None:
artifacts = {}
if isinstance(inputs, dict):
inputs = update_inputs_dict(inputs, artifacts)
elif isinstance(inputs, List):
inputs = [update_inputs_dict(inp, artifacts) for inp in inputs]
return inputs
def update_inputs_dict(inputs: dict, artifacts: Dict[str, Any]) -> dict:
for key, value in artifacts.items():
if key == "repr":
continue
@ -188,7 +200,7 @@ async def generate_result(built_object: Union[Chain, VectorStore, Runnable], inp
class Result(BaseModel):
result: Union[dict, List[dict], str, List[str], AIMessage]
result: Any
session_id: str

15
src/backend/langflow/utils/validate.py
View file

@ -2,7 +2,7 @@ import ast
import contextlib
import importlib
from types import FunctionType
from typing import Dict
from typing import Dict, List, Optional, Union
from langflow.field_typing.constants import CUSTOM_COMPONENT_SUPPORTED_TYPES
@ -260,14 +260,13 @@ def get_default_imports(code_string):
"""
Returns a dictionary of default imports for the dynamic class constructor.
"""
typing_module = importlib.import_module("typing")
default_imports = {
"Optional": typing_module.Optional,
"List": typing_module.List,
"Dict": typing_module.Dict,
"Union": typing_module.Union,
}
default_imports = {
"Optional": Optional,
"List": List,
"Dict": Dict,
"Union": Union,
}
langflow_imports = list(CUSTOM_COMPONENT_SUPPORTED_TYPES.keys())
necessary_imports = find_names_in_code(code_string, langflow_imports)
langflow_module = importlib.import_module("langflow.field_typing")

9
src/backend/langflow/worker.py
View file

@ -1,14 +1,13 @@
from typing import TYPE_CHECKING, Any, Dict, Optional
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from asgiref.sync import async_to_sync
from celery.exceptions import SoftTimeLimitExceeded # type: ignore
from loguru import logger
from rich import print
from langflow.core.celery_app import celery_app
from langflow.processing.process import Result, generate_result, process_inputs
from langflow.services.deps import get_session_service
from langflow.services.manager import initialize_session_service
from loguru import logger
from rich import print
if TYPE_CHECKING:
from langflow.graph.vertex.base import Vertex
@ -35,7 +34,7 @@ def build_vertex(self, vertex: "Vertex") -> "Vertex":
@celery_app.task(acks_late=True)
def process_graph_cached_task(
data_graph: Dict[str, Any],
inputs: Optional[dict] = None,
inputs: Optional[Union[dict, List[dict]]] = None,
clear_cache=False,
session_id=None,
) -> Dict[str, Any]:

2
src/frontend/src/utils/styleUtils.ts
View file

@ -129,12 +129,12 @@ import { NotionIcon } from "../icons/Notion";
import { OpenAiIcon } from "../icons/OpenAi";
import { PineconeIcon } from "../icons/Pinecone";
import { QDrantIcon } from "../icons/QDrant";
import { WeaviateIcon } from "../icons/Weaviate";
import { SearxIcon } from "../icons/Searx";
import { ShareIcon } from "../icons/Share";
import { Share2Icon } from "../icons/Share2";
import SvgSlackIcon from "../icons/Slack/SlackIcon";
import { VertexAIIcon } from "../icons/VertexAI";
import { WeaviateIcon } from "../icons/Weaviate";
import SvgWikipedia from "../icons/Wikipedia/Wikipedia";
import SvgWolfram from "../icons/Wolfram/Wolfram";
import { HackerNewsIcon } from "../icons/hackerNews";

197
src/frontend/tests/onlyFront/assets/flowtest.json
View file

File diff suppressed because one or more lines are too long

26
tests/test_custom_component.py
View file

@ -3,14 +3,10 @@ import types
from uuid import uuid4
import pytest
from fastapi import HTTPException
from langflow.interface.custom.base import CustomComponent
from langflow.interface.custom.code_parser.code_parser import (CodeParser,
CodeSyntaxError)
from langflow.interface.custom.custom_component.component import (
Component, ComponentCodeNullError)
from langflow.interface.custom.utils import (build_custom_component_template,
create_and_validate_component)
from langflow.interface.custom.code_parser.code_parser import CodeParser, CodeSyntaxError
from langflow.interface.custom.custom_component.component import Component, ComponentCodeNullError
from langflow.interface.custom.utils import build_custom_component_template
from langflow.services.database.models.flow import Flow, FlowCreate
code_default = """
@ -368,16 +364,6 @@ def test_component_get_code_tree_syntax_error():
component.get_code_tree(component.code)
def test_custom_component_class_template_validation_no_code():
"""
Test the _class_template_validation method of the CustomComponent class
raises the HTTPException when the code is None.
"""
custom_component = CustomComponent(code=None, function_entrypoint_name="build")
with pytest.raises(HTTPException):
custom_component._class_template_validation(custom_component.code)
def test_custom_component_get_code_tree_syntax_error():
"""
Test the get_code_tree method of the CustomComponent class
@ -535,12 +521,12 @@ def test_build_config_field_value_keys(component):
def test_create_and_validate_component_valid_code(test_component_code):
component = create_and_validate_component(test_component_code)
component = CustomComponent(code=test_component_code)
assert isinstance(component, CustomComponent)
def test_build_langchain_template_custom_component_valid_code(test_component_code):
component = create_and_validate_component(test_component_code)
component = CustomComponent(code=test_component_code)
frontend_node = build_custom_component_template(component)
assert isinstance(frontend_node, dict)
template = frontend_node["template"]
@ -554,7 +540,7 @@ def test_build_langchain_template_custom_component_valid_code(test_component_cod
def test_build_langchain_template_custom_component_templatefield(test_component_with_templatefield_code):
component = create_and_validate_component(test_component_with_templatefield_code)
component = CustomComponent(code=test_component_with_templatefield_code)
frontend_node = build_custom_component_template(component)
assert isinstance(frontend_node, dict)
template = frontend_node["template"]