feat: start using dev mode flag and add exception filter in logger (#3260)

* refactor: reorganize logger module and add setup.py for logging configuration

* refactor: update logger import path to align with recent module restructuring

* refactor: add logging configuration parameter to Graph initialization for improved logging setup flexibility

* feat: create logging init module for improved logger configuration and management

* refactor: update Settings class to include development mode flag and associated validator for enhanced configuration management

* refactor: enhance logger.py with DEV mode handling and TypedDict for log configuration settings

* feat: add settings module with DEV mode flag and helper functions for setting development state

* refactor: update flow assertions in tests to check for Data object type instead of Flow object type

* feat: add deepcopy method to Graph class to copy start and end components, ensuring proper graph cloning functionality

* feat: implement deepcopy method in Component class for proper cloning of inputs and configuration attributes

* feat: enhance attribute access in Component class to support backwards-compatible private attributes retrieval

* feat: improve test assertion for list_flows in custom component to display types of returned objects for better debugging

* feat: refactor imports in constants.py and remove redundant Data class definition for cleaner code structure

* feat: refactor imports in logger.py to include NotRequired from typing_extensions for better type hinting support

src/backend/base/langflow/__main__.py

@@ -26,7 +26,7 @@ from langflow.services.database.utils import session_getter
 from langflow.services.deps import get_db_service, get_settings_service, session_scope
 from langflow.services.settings.constants import DEFAULT_SUPERUSER
 from langflow.services.utils import initialize_services
-from langflow.utils.logger import configure, logger
+from langflow.logging.logger import configure, logger
 from langflow.utils.util import update_settings
 
 console = Console()

src/backend/base/langflow/api/log_router.py

@@ -5,7 +5,7 @@ from typing import List, Any
 from fastapi import APIRouter, Query, HTTPException, Request
 from fastapi.responses import JSONResponse, StreamingResponse
 from http import HTTPStatus
-from langflow.utils.logger import log_buffer
+from langflow.logging.logger import log_buffer
 
 log_router = APIRouter(tags=["Log"])
 

src/backend/base/langflow/custom/custom_component/component.py

@@ -1,4 +1,5 @@
 import inspect
+from copy import deepcopy
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, List, Optional, Union, get_type_hints
 from uuid import UUID
 
@@ -34,6 +35,7 @@ class Component(CustomComponent):
     def __init__(self, **kwargs):
         # if key starts with _ it is a config
         # else it is an input
+
         inputs = {}
         config = {}
         for key, value in kwargs.items():
@@ -53,6 +55,8 @@ class Component(CustomComponent):
         config = config or {}
         if "_id" not in config:
             config |= {"_id": f"{self.__class__.__name__}-{nanoid.generate(size=5)}"}
+        self.__inputs = inputs
+        self.__config = config
         super().__init__(**config)
         if hasattr(self, "_trace_type"):
             self.trace_type = self._trace_type
@@ -66,6 +70,24 @@ class Component(CustomComponent):
         self._set_output_types()
         self.set_class_code()
 
+    def __deepcopy__(self, memo):
+        if id(self) in memo:
+            return memo[id(self)]
+        kwargs = deepcopy(self.__config)
+        kwargs["inputs"] = deepcopy(self.__inputs)
+        new_component = type(self)(**kwargs)
+        new_component._code = self._code
+        new_component._outputs = self._outputs
+        new_component._inputs = self._inputs
+        new_component._edges = self._edges
+        new_component._components = self._components
+        new_component._parameters = self._parameters
+        new_component._attributes = self._attributes
+        new_component._output_logs = self._output_logs
+        new_component._logs = self._logs
+        memo[id(self)] = new_component
+        return new_component
+
     def set_class_code(self):
         # Get the source code of the calling class
         if self._code:
@@ -331,6 +353,8 @@ class Component(CustomComponent):
             return self.__dict__["_inputs"][name].value
         if name in BACKWARDS_COMPATIBLE_ATTRIBUTES:
             return self.__dict__[f"_{name}"]
+        if name.startswith("_") and name[1:] in BACKWARDS_COMPATIBLE_ATTRIBUTES:
+            return self.__dict__[name]
         raise AttributeError(f"{name} not found in {self.__class__.__name__}")
 
     def _set_input_value(self, name: str, value: Any):

src/backend/base/langflow/field_typing/constants.py

@@ -13,10 +13,11 @@ from langchain_core.memory import BaseMemory
 from langchain_core.output_parsers import BaseOutputParser
 from langchain_core.prompts import BasePromptTemplate, ChatPromptTemplate, PromptTemplate
 from langchain_core.retrievers import BaseRetriever
-from langchain_core.tools import Tool, BaseTool
+from langchain_core.tools import BaseTool, Tool
 from langchain_core.vectorstores import VectorStore, VectorStoreRetriever
 from langchain_text_splitters import TextSplitter
 
+from langflow.schema.data import Data
 from langflow.schema.message import Message
 
 NestedDict: TypeAlias = Dict[str, Union[str, Dict]]
@@ -33,10 +34,6 @@ class Object:
     pass
 
 
-class Data:
-    pass
-
-
 class Code:
     pass
 

src/backend/base/langflow/graph/graph/base.py

@@ -2,12 +2,12 @@ import asyncio
 import copy
 import json
 import uuid
+import warnings
 from collections import defaultdict, deque
 from datetime import datetime, timezone
 from functools import partial
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Tuple, Type, Union
-import warnings
 
 import nest_asyncio
 from loguru import logger
@@ -24,6 +24,7 @@ from langflow.graph.schema import InterfaceComponentTypes, RunOutputs
 from langflow.graph.vertex.base import Vertex, VertexStates
 from langflow.graph.vertex.schema import NodeData
 from langflow.graph.vertex.types import ComponentVertex, InterfaceVertex, StateVertex
+from langflow.logging.logger import LogConfig, configure
 from langflow.schema import Data
 from langflow.schema.schema import INPUT_FIELD_NAME, InputType
 from langflow.services.cache.utils import CacheMiss
@@ -47,6 +48,7 @@ class Graph:
         flow_id: Optional[str] = None,
         flow_name: Optional[str] = None,
         user_id: Optional[str] = None,
+        log_config: Optional[LogConfig] = None,
     ) -> None:
         """
         Initializes a new instance of the Graph class.
@@ -56,6 +58,11 @@ class Graph:
             edges (List[Dict[str, str]]): A list of dictionaries representing the edges of the graph.
             flow_id (Optional[str], optional): The ID of the flow. Defaults to None.
         """
+        if not log_config:
+            log_config = {"disable": False}
+        configure(**log_config)
+        self._start = start
+        self._end = end
         self._prepared = False
         self._runs = 0
         self._updates = 0
@@ -803,7 +810,6 @@ class Graph:
             "vertices_layers": self.vertices_layers,
             "vertices_to_run": self.vertices_to_run,
             "stop_vertex": self.stop_vertex,
-            "vertex_map": self.vertex_map,
             "_run_queue": self._run_queue,
             "_first_layer": self._first_layer,
             "_vertices": self._vertices,
@@ -814,6 +820,39 @@ class Graph:
             "_sorted_vertices_layers": self._sorted_vertices_layers,
         }
 
+    def __deepcopy__(self, memo):
+        # Check if we've already copied this instance
+        if id(self) in memo:
+            return memo[id(self)]
+
+        if self._start is not None and self._end is not None:
+            # Deep copy start and end components
+            start_copy = copy.deepcopy(self._start, memo)
+            end_copy = copy.deepcopy(self._end, memo)
+            new_graph = type(self)(
+                start_copy,
+                end_copy,
+                copy.deepcopy(self.flow_id, memo),
+                copy.deepcopy(self.flow_name, memo),
+                copy.deepcopy(self.user_id, memo),
+            )
+        else:
+            # Create a new graph without start and end, but copy flow_id, flow_name, and user_id
+            new_graph = type(self)(
+                None,
+                None,
+                copy.deepcopy(self.flow_id, memo),
+                copy.deepcopy(self.flow_name, memo),
+                copy.deepcopy(self.user_id, memo),
+            )
+            # Deep copy vertices and edges
+            new_graph.add_nodes_and_edges(copy.deepcopy(self._vertices, memo), copy.deepcopy(self._edges, memo))
+
+        # Store the newly created object in memo
+        memo[id(self)] = new_graph
+
+        return new_graph
+
     def __setstate__(self, state):
         run_manager = state["run_manager"]
         if isinstance(run_manager, RunnableVerticesManager):
@@ -821,6 +860,7 @@ class Graph:
         else:
             state["run_manager"] = RunnableVerticesManager.from_dict(run_manager)
         self.__dict__.update(state)
+        self.vertex_map = {vertex.id: vertex for vertex in self.vertices}
         self.state_manager = GraphStateManager()
         self.tracing_service = get_tracing_service()
         self.set_run_id(self._run_id)

src/backend/base/langflow/load/load.py

@@ -8,7 +8,7 @@ from loguru import logger
 from langflow.graph import Graph
 from langflow.graph.schema import RunOutputs
 from langflow.processing.process import process_tweaks, run_graph
-from langflow.utils.logger import configure
+from langflow.logging.logger import configure
 from langflow.utils.util import update_settings
 
 

src/backend/base/langflow/logging/__init__.py

@@ -0,0 +1,4 @@
+from .logger import configure, logger
+from .setup import disable_logging, enable_logging
+
+__all__ = ["configure", "logger", "disable_logging", "enable_logging"]

src/backend/base/langflow/logging/logger.py

@@ -2,15 +2,18 @@ import json
 import logging
 import os
 import sys
-from pathlib import Path
 from collections import deque
+from pathlib import Path
 from threading import Lock, Semaphore
-from typing import Optional
+from typing import Optional, TypedDict
 
 import orjson
 from loguru import logger
 from platformdirs import user_cache_dir
 from rich.logging import RichHandler
+from typing_extensions import NotRequired
+
+from langflow.settings import DEV
 
 VALID_LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
 
@@ -129,6 +132,15 @@ def serialize_log(record):
 
 def patching(record):
     record["extra"]["serialized"] = serialize_log(record)
+    if DEV is False:
+        record.pop("exception", None)
+
+
+class LogConfig(TypedDict):
+    log_level: NotRequired[str]
+    log_file: NotRequired[Path]
+    disable: NotRequired[bool]
+    log_env: NotRequired[str]
 
 
 def configure(

src/backend/base/langflow/logging/setup.py

@@ -0,0 +1,16 @@
+from loguru import logger
+
+LOGGING_CONFIGURED = False
+
+
+def disable_logging():
+    global LOGGING_CONFIGURED
+    if not LOGGING_CONFIGURED:
+        logger.disable("langflow")
+        LOGGING_CONFIGURED = True
+
+
+def enable_logging():
+    global LOGGING_CONFIGURED
+    logger.enable("langflow")
+    LOGGING_CONFIGURED = True

src/backend/base/langflow/main.py

@@ -31,7 +31,7 @@ from langflow.interface.utils import setup_llm_caching
 from langflow.services.deps import get_cache_service, get_settings_service, get_telemetry_service
 from langflow.services.plugins.langfuse_plugin import LangfuseInstance
 from langflow.services.utils import initialize_services, teardown_services
-from langflow.utils.logger import configure
+from langflow.logging.logger import configure
 
 # Ignore Pydantic deprecation warnings from Langchain
 warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20)

src/backend/base/langflow/server.py

@@ -6,7 +6,7 @@ from gunicorn import glogging  # type: ignore
 from gunicorn.app.base import BaseApplication  # type: ignore
 from uvicorn.workers import UvicornWorker
 
-from langflow.utils.logger import InterceptHandler  # type: ignore
+from langflow.logging.logger import InterceptHandler  # type: ignore
 
 
 class LangflowUvicornWorker(UvicornWorker):

src/backend/base/langflow/services/cache/factory.py

@@ -3,7 +3,7 @@ from typing import TYPE_CHECKING
 from langflow.services.cache.disk import AsyncDiskCache
 from langflow.services.cache.service import AsyncInMemoryCache, CacheService, RedisCache, ThreadingInMemoryCache
 from langflow.services.factory import ServiceFactory
-from langflow.utils.logger import logger
+from langflow.logging.logger import logger
 
 if TYPE_CHECKING:
     from langflow.services.settings.service import SettingsService

src/backend/base/langflow/services/settings/base.py

@@ -66,6 +66,7 @@ class Settings(BaseSettings):
     """Define if langflow database should be saved in LANGFLOW_CONFIG_DIR or in the langflow directory (i.e. in the package directory)."""
 
     dev: bool = False
+    """If True, Langflow will run in development mode."""
     database_url: Optional[str] = None
     """Database URL for Langflow. If not provided, Langflow will use a SQLite database."""
     pool_size: int = 10
@@ -151,6 +152,14 @@ class Settings(BaseSettings):
     vertex_builds_storage_enabled: bool = True
     """If set to True, Langflow will keep track of each vertex builds (outputs) in the UI for any flow."""
 
+    @field_validator("dev")
+    @classmethod
+    def set_dev(cls, value):
+        from langflow.settings import set_dev
+
+        set_dev(value)
+        return value
+
     @field_validator("user_agent", mode="after")
     @classmethod
     def set_user_agent(cls, value):

src/backend/base/langflow/settings.py

@@ -0,0 +1,10 @@
+DEV = False
+
+
+def _set_dev(value):
+    global DEV
+    DEV = value
+
+
+def set_dev(value):
+    _set_dev(value)

src/backend/base/langflow/utils/util.py

@@ -12,7 +12,7 @@ from langflow.schema import Data
 from langflow.services.deps import get_settings_service
 from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
 from langflow.utils import constants
-from langflow.utils.logger import logger
+from langflow.logging.logger import logger
 
 
 def unescape_string(s: str):

src/backend/tests/unit/test_custom_component_with_client.py

@@ -1,7 +1,7 @@
 import pytest
 
 from langflow.custom.custom_component.custom_component import CustomComponent
-from langflow.services.database.models.flow import Flow
+from langflow.field_typing.constants import Data
 
 
 @pytest.fixture
@@ -20,4 +20,6 @@ def component(client, active_user):
 
 def test_list_flows_flow_objects(component):
     flows = component.list_flows()
-    assert all(isinstance(flow, Flow) for flow in flows)
+    are_flows = [isinstance(flow, Data) for flow in flows]
+    flow_types = [type(flow) for flow in flows]
+    assert all(are_flows), f"Expected all flows to be Data objects, got {flow_types}"

src/backend/tests/unit/test_logger.py

@@ -2,7 +2,7 @@ import pytest
 import os
 import json
 from unittest.mock import patch
-from langflow.utils.logger import SizedLogBuffer
+from langflow.logging.logger import SizedLogBuffer
 
 
 @pytest.fixture