From 0b535c54f540b828a53c6301ffdcd15eba2bb09d Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 27 Sep 2023 08:48:21 -0300 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=94=A7=20chore(=5F=5Fmain=5F=5F.py):?= =?UTF-8?q?=20update=20Typer=20configuration=20to=20treat=20no=20arguments?= =?UTF-8?q?=20as=20help=20to=20improve=20usability=20=F0=9F=94=A7=20chore(?= =?UTF-8?q?=5F=5Fmain=5F=5F.py):=20update=20docstring=20for=20run()=20func?= =?UTF-8?q?tion=20to=20improve=20clarity=20=F0=9F=94=A7=20chore(=5F=5Fmain?= =?UTF-8?q?=5F=5F.py):=20update=20docstring=20for=20superuser()=20function?= =?UTF-8?q?=20to=20improve=20clarity=20=F0=9F=94=A7=20chore(=5F=5Fmain=5F?= =?UTF-8?q?=5F.py):=20update=20migration()=20function=20to=20default=20to?= =?UTF-8?q?=20test=20mode=20=F0=9F=94=A7=20chore(manager.py):=20remove=20u?= =?UTF-8?q?nnecessary=20logging=20of=20database=20URL=20in=20run=5Fmigrati?= =?UTF-8?q?ons()=20function=20=F0=9F=94=A7=20chore(manager.py):=20add=20mi?= =?UTF-8?q?ssing=20import=20statement=20for=20logger=20in=20check=5Fcelery?= =?UTF-8?q?=5Favailability()=20function=20=F0=9F=94=A7=20chore(manager.py)?= =?UTF-8?q?:=20update=20configure()=20function=20to=20be=20called=20in=20c?= =?UTF-8?q?heck=5Fcelery=5Favailability()=20function=20=F0=9F=94=A7=20chor?= =?UTF-8?q?e(logger.py):=20update=20configure()=20function=20to=20use=20en?= =?UTF-8?q?vironment=20variable=20for=20log=20level=20if=20available=20?= =?UTF-8?q?=F0=9F=94=A7=20chore(logger.py):=20update=20configure()=20funct?= =?UTF-8?q?ion=20to=20use=20default=20log=20file=20location=20if=20not=20p?= =?UTF-8?q?rovided?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/__main__.py | 16 ++++-- .../langflow/services/database/manager.py | 4 +- src/backend/langflow/services/task/manager.py | 10 ++-- src/backend/langflow/utils/logger.py | 52 ++++++++++++++----- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/src/backend/langflow/__main__.py b/src/backend/langflow/__main__.py index 6701d27bc..f42f44a9e 100644 --- a/src/backend/langflow/__main__.py +++ b/src/backend/langflow/__main__.py @@ -23,7 +23,7 @@ from rich.table import Table console = Console() -app = typer.Typer() +app = typer.Typer(no_args_is_help=True) def get_number_of_workers(workers=None): @@ -141,7 +141,7 @@ def run( ), ): """ - Run the Langflow server. + Run the Langflow. """ # override env variables with .env file if env_file: @@ -299,7 +299,14 @@ def superuser( password: str = typer.Option( ..., prompt=True, hide_input=True, help="Password for the superuser." ), + log_level: str = typer.Option( + "critical", help="Logging level.", envvar="LANGFLOW_LOG_LEVEL" + ), ): + """ + Create a superuser. + """ + configure(log_level=log_level) initialize_services() db_service = get_db_service() with session_getter(db_service) as session: @@ -321,7 +328,10 @@ def superuser( @app.command() -def migration(test: bool = typer.Option(False, help="Run migrations in test mode.")): +def migration(test: bool = typer.Option(True, help="Run migrations in test mode.")): + """ + Run or test migrations. + """ initialize_services() db_service = get_db_service() if not test: diff --git a/src/backend/langflow/services/database/manager.py b/src/backend/langflow/services/database/manager.py index ca7f34d10..3a388f694 100644 --- a/src/backend/langflow/services/database/manager.py +++ b/src/backend/langflow/services/database/manager.py @@ -94,9 +94,7 @@ class DatabaseService(Service): return True def run_migrations(self): - logger.info( - f"Running DB migrations in {self.script_location} on {self.database_url}" - ) + logger.info(f"Running DB migrations in {self.script_location}") alembic_cfg = Config() alembic_cfg.set_main_option("script_location", str(self.script_location)) alembic_cfg.set_main_option("sqlalchemy.url", self.database_url) diff --git a/src/backend/langflow/services/task/manager.py b/src/backend/langflow/services/task/manager.py index ab74d916e..422b34faa 100644 --- a/src/backend/langflow/services/task/manager.py +++ b/src/backend/langflow/services/task/manager.py @@ -1,4 +1,5 @@ from typing import Any, Callable, Coroutine, Union +from langflow.utils.logger import configure from loguru import logger from langflow.services.base import Service from langflow.services.task.backends.anyio import AnyIOBackend @@ -7,18 +8,19 @@ from langflow.services.task.utils import get_celery_worker_status def check_celery_availability(): - from langflow.worker import celery_app - try: + from langflow.worker import celery_app + status = get_celery_worker_status(celery_app) logger.debug(f"Celery status: {status}") - except Exception as e: - logger.error(f"An error occurred: {e}") + except Exception as exc: + logger.debug(f"Celery not available: {exc}") status = {"availability": None} return status try: + configure() status = check_celery_availability() USE_CELERY = status.get("availability") is not None diff --git a/src/backend/langflow/utils/logger.py b/src/backend/langflow/utils/logger.py index 1f616486b..05b7adc0e 100644 --- a/src/backend/langflow/utils/logger.py +++ b/src/backend/langflow/utils/logger.py @@ -2,12 +2,34 @@ from typing import Optional from loguru import logger from pathlib import Path from rich.logging import RichHandler +import os +import orjson +import appdirs -def configure(log_level: str = "DEBUG", log_file: Optional[Path] = None): - log_format = "{time:HH:mm:ss} - {level: <8} - {message}" +def serialize(record): + subset = { + "timestamp": record["time"].timestamp(), + "message": record["message"], + "level": record["level"].name, + "module": record["module"], + } + return orjson.dumps(subset) + + +def patching(record): + record["extra"]["serialized"] = serialize(record) + + +def configure(log_level: str = "INFO", log_file: Optional[Path] = None): + if os.getenv("LANGFLOW_LOG_LEVEL"): + log_level = os.getenv("LANGFLOW_LOG_LEVEL") + # Human-readable + log_format = "{time:YYYY-MM-DD HH:mm:ss} - {level: <8} - {module} - {message}" + + # log_format = log_format_dev if log_level.upper() == "DEBUG" else log_format_prod logger.remove() # Remove default handlers - + logger.patch(patching) # Configure loguru to use RichHandler logger.configure( handlers=[ @@ -19,17 +41,21 @@ def configure(log_level: str = "DEBUG", log_file: Optional[Path] = None): ] ) - if log_file: - log_file = Path(log_file) - log_file.parent.mkdir(parents=True, exist_ok=True) + if not log_file: + cache_dir = Path(appdirs.user_cache_dir("langflow")) + log_file = cache_dir / "langflow.log" - logger.add( - sink=str(log_file), - level=log_level.upper(), - format=log_format, - rotation="10 MB", # Log rotation based on file size - ) + log_file = Path(log_file) + log_file.parent.mkdir(parents=True, exist_ok=True) - logger.info(f"Logger set up with log level: {log_level}") + logger.add( + sink=str(log_file), + level=log_level.upper(), + format=log_format, + rotation="10 MB", # Log rotation based on file size + serialize=True, + ) + + logger.debug(f"Logger set up with log level: {log_level}") if log_file: logger.info(f"Log file: {log_file}") From 0b2b074e07c8da32f1eb9f6d968eb80e98519ada Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 27 Sep 2023 08:51:17 -0300 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=94=A7=20chore(logger.py):=20refactor?= =?UTF-8?q?=20configure=20function=20to=20improve=20readability=20and=20ad?= =?UTF-8?q?d=20support=20for=20configurable=20log=20level=20=F0=9F=94=92?= =?UTF-8?q?=20chore(logger.py):=20add=20VALID=5FLOG=5FLEVELS=20constant=20?= =?UTF-8?q?to=20define=20valid=20log=20levels=20for=20better=20maintainabi?= =?UTF-8?q?lity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/utils/logger.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/backend/langflow/utils/logger.py b/src/backend/langflow/utils/logger.py index 05b7adc0e..e49fa4bfd 100644 --- a/src/backend/langflow/utils/logger.py +++ b/src/backend/langflow/utils/logger.py @@ -7,6 +7,9 @@ import orjson import appdirs +VALID_LOG_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] + + def serialize(record): subset = { "timestamp": record["time"].timestamp(), @@ -21,9 +24,11 @@ def patching(record): record["extra"]["serialized"] = serialize(record) -def configure(log_level: str = "INFO", log_file: Optional[Path] = None): - if os.getenv("LANGFLOW_LOG_LEVEL"): +def configure(log_level: Optional[str] = None, log_file: Optional[Path] = None): + if os.getenv("LANGFLOW_LOG_LEVEL") in VALID_LOG_LEVELS and not log_level: log_level = os.getenv("LANGFLOW_LOG_LEVEL") + elif not log_level: + log_level = "INFO" # Human-readable log_format = "{time:YYYY-MM-DD HH:mm:ss} - {level: <8} - {module} - {message}" From ede745c1ddc36b27e8298e3ce598467369b63f50 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 27 Sep 2023 08:54:11 -0300 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=90=9B=20fix(custom=5Fcomponent.py):?= =?UTF-8?q?=20improve=20error=20message=20formatting=20for=20type=20hint?= =?UTF-8?q?=20error=20in=20build=20method=20=F0=9F=90=9B=20fix(logger.py):?= =?UTF-8?q?=20fix=20conditional=20statement=20for=20log=5Flevel=20configur?= =?UTF-8?q?ation=20to=20properly=20handle=20None=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/custom/custom_component.py | 5 ++++- src/backend/langflow/utils/logger.py | 10 +++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/backend/langflow/interface/custom/custom_component.py b/src/backend/langflow/interface/custom/custom_component.py index b0c4f8752..ccc0b08b2 100644 --- a/src/backend/langflow/interface/custom/custom_component.py +++ b/src/backend/langflow/interface/custom/custom_component.py @@ -102,7 +102,10 @@ class CustomComponent(Component, extra=Extra.allow): status_code=400, detail={ "error": "Type hint Error", - "traceback": "Prompt type is not supported in the build method. Try using PromptTemplate instead.", + "traceback": ( + "Prompt type is not supported in the build method." + " Try using PromptTemplate instead." + ), }, ) return args diff --git a/src/backend/langflow/utils/logger.py b/src/backend/langflow/utils/logger.py index e49fa4bfd..ebc468e7e 100644 --- a/src/backend/langflow/utils/logger.py +++ b/src/backend/langflow/utils/logger.py @@ -1,3 +1,4 @@ +from math import log from typing import Optional from loguru import logger from pathlib import Path @@ -25,12 +26,15 @@ def patching(record): def configure(log_level: Optional[str] = None, log_file: Optional[Path] = None): - if os.getenv("LANGFLOW_LOG_LEVEL") in VALID_LOG_LEVELS and not log_level: + if os.getenv("LANGFLOW_LOG_LEVEL") in VALID_LOG_LEVELS and log_level is None: log_level = os.getenv("LANGFLOW_LOG_LEVEL") - elif not log_level: + if log_level is None: log_level = "INFO" # Human-readable - log_format = "{time:YYYY-MM-DD HH:mm:ss} - {level: <8} - {module} - {message}" + log_format = ( + "{time:YYYY-MM-DD HH:mm:ss} - " + "{level: <8} - {module} - {message}" + ) # log_format = log_format_dev if log_level.upper() == "DEBUG" else log_format_prod logger.remove() # Remove default handlers From 05c73d4664a1fd0c157e308b2d7a88f590971833 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 27 Sep 2023 09:10:42 -0300 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=A7=20chore(Makefile):=20remove=20?= =?UTF-8?q?unnecessary=20parallel=20test=20execution=20flag=20to=20simplif?= =?UTF-8?q?y=20test=20command=20=F0=9F=94=A7=20chore(Makefile):=20remove?= =?UTF-8?q?=20unnecessary=20parallel=20test=20execution=20flag=20to=20simp?= =?UTF-8?q?lify=20test=20command=20=F0=9F=94=A7=20chore(Makefile):=20remov?= =?UTF-8?q?e=20unnecessary=20parallel=20test=20execution=20flag=20to=20sim?= =?UTF-8?q?plify=20test=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 32198730f..1fbd7403c 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ coverage: tests: @make install_backend - poetry run pytest tests -n auto + poetry run pytest tests format: poetry run black .