jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Update settings attributes and remove config.yaml references (#1975)

Browse source 
* chore: update settings attributes and remove config.yaml references

* Remove config.yaml references and update settings attributes
This commit is contained in:
Gabriel Luiz Freitas Almeida 2024-05-26 07:24:21 -07:00 • committed by GitHub
commit e23544a1b2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 83 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -65,7 +65,6 @@ Each option is detailed below:
- `--workers`: Sets the number of worker processes. Can be set using the `LANGFLOW_WORKERS` environment variable. The default is `1`.
- `--timeout`: Sets the worker timeout in seconds. The default is `60`.
- `--port`: Sets the port to listen on. Can be set using the `LANGFLOW_PORT` environment variable. The default is `7860`.
- `--config`: Defines the path to the configuration file. The default is `config.yaml`.
- `--env-file`: Specifies the path to the .env file containing environment variables. The default is `.env`.
- `--log-level`: Defines the logging level. Can be set using the `LANGFLOW_LOG_LEVEL` environment variable. The default is `critical`.
- `--components-path`: Specifies the path to the directory containing custom components. Can be set using the `LANGFLOW_COMPONENTS_PATH` environment variable. The default is `langflow/components`.

1
docs/docs/administration/cli.mdx
View file

@ -19,7 +19,6 @@ Each option is detailed below:
- `--workers`: Sets the number of worker processes. Can be set using the `LANGFLOW_WORKERS` environment variable. The default is `1`.
- `--timeout`: Sets the worker timeout in seconds. The default is `60`.
- `--port`: Sets the port to listen on. Can be set using the `LANGFLOW_PORT` environment variable. The default is `7860`.
- `--config`: Defines the path to the configuration file. The default is `config.yaml`.
- `--env-file`: Specifies the path to the .env file containing environment variables. The default is `.env`.
- `--log-level`: Defines the logging level. Can be set using the `LANGFLOW_LOG_LEVEL` environment variable. The default is `critical`.
- `--components-path`: Specifies the path to the directory containing custom components. Can be set using the `LANGFLOW_COMPONENTS_PATH` environment variable. The default is `langflow/components`.

2
src/backend/base/langflow/__main__.py
View file

@ -84,7 +84,6 @@ def run(
help="Path to the directory containing custom components.",
envvar="LANGFLOW_COMPONENTS_PATH",
),
config: str = typer.Option(Path(__file__).parent / "config.yaml", help="Path to the configuration file."),
# .env file param
env_file: Path = typer.Option(None, help="Path to the .env file containing environment variables."),
log_level: str = typer.Option("critical", help="Logging level.", envvar="LANGFLOW_LOG_LEVEL"),
@ -133,7 +132,6 @@ def run(
load_dotenv(env_file, override=True)
update_settings(
config,
dev=dev,
remove_api_keys=remove_api_keys,
cache=cache,

2
src/backend/base/langflow/api/v1/endpoints.py
View file

@ -43,7 +43,7 @@ def get_all(
logger.debug("Building langchain types dict")
try:
all_types_dict = get_all_types_dict(settings_service.settings.COMPONENTS_PATH)
all_types_dict = get_all_types_dict(settings_service.settings.components_path)
return all_types_dict
except Exception as exc:
logger.exception(exc)

2
src/backend/base/langflow/api/v1/flows.py
View file

@ -127,7 +127,7 @@ def update_flow(
if not db_flow:
raise HTTPException(status_code=404, detail="Flow not found")
flow_data = flow.model_dump(exclude_unset=True)
if settings_service.settings.REMOVE_API_KEYS:
if settings_service.settings.remove_api_keys:
flow_data = remove_api_keys(flow_data)
for key, value in flow_data.items():
if value is not None:

2
src/backend/base/langflow/initial_setup/setup.py
View file

@ -206,7 +206,7 @@ def create_starter_folder(session):
def create_or_update_starter_projects():
components_paths = get_settings_service().settings.COMPONENTS_PATH
components_paths = get_settings_service().settings.components_path
try:
all_types_dict = get_all_components(components_paths, as_dict=True)
except Exception as e:

2
src/backend/base/langflow/interface/listing.py
View file

@ -21,7 +21,7 @@ class AllTypesDict(LazyLoadDictBase):
from langflow.interface.types import get_all_types_dict
settings_service = get_settings_service()
return get_all_types_dict(settings_service.settings.COMPONENTS_PATH)
return get_all_types_dict(settings_service.settings.components_path)
lazy_load_dict = AllTypesDict()

10
src/backend/base/langflow/interface/utils.py
View file

@ -7,12 +7,13 @@ from typing import Dict
import yaml
from docstring_parser import parse
from langchain_core.language_models import BaseLanguageModel
from loguru import logger
from PIL.Image import Image
from langflow.services.chat.config import ChatConfig
from langflow.services.deps import get_settings_service
from langflow.utils.util import format_dict, get_base_classes, get_default_factory
from loguru import logger
from PIL.Image import Image
from langchain_core.language_models import BaseLanguageModel
def load_file_into_dict(file_path: str) -> dict:
@ -95,13 +96,14 @@ def setup_llm_caching():
try:
set_langchain_cache(settings_service.settings)
except ImportError:
logger.warning(f"Could not import {settings_service.settings.CACHE_TYPE}. ")
logger.warning(f"Could not import {settings_service.settings.cache_type}. ")
except Exception as exc:
logger.warning(f"Could not setup LLM caching. Error: {exc}")
def set_langchain_cache(settings):
from langchain.globals import set_llm_cache
from langflow.interface.importing.utils import import_class
if cache_type := os.getenv("LANGFLOW_LANGCHAIN_CACHE"):

16
src/backend/base/langflow/services/cache/factory.py vendored
View file

@ -16,14 +16,14 @@ class CacheServiceFactory(ServiceFactory):
# Here you would have logic to create and configure a CacheService
# based on the settings_service
if settings_service.settings.CACHE_TYPE == "redis":
if settings_service.settings.cache_type == "redis":
logger.debug("Creating Redis cache")
redis_cache = RedisCache(
host=settings_service.settings.REDIS_HOST,
port=settings_service.settings.REDIS_PORT,
db=settings_service.settings.REDIS_DB,
url=settings_service.settings.REDIS_URL,
expiration_time=settings_service.settings.REDIS_CACHE_EXPIRE,
host=settings_service.settings.redis_host,
port=settings_service.settings.redis_port,
db=settings_service.settings.redis_db,
url=settings_service.settings.redis_url,
expiration_time=settings_service.settings.redis_cache_expire,
)
if redis_cache.is_connected():
logger.debug("Redis cache is connected")
@ -31,7 +31,7 @@ class CacheServiceFactory(ServiceFactory):
logger.warning("Redis cache is not connected, falling back to in-memory cache")
return ThreadingInMemoryCache()
elif settings_service.settings.CACHE_TYPE == "memory":
elif settings_service.settings.cache_type == "memory":
return ThreadingInMemoryCache()
elif settings_service.settings.CACHE_TYPE == "async":
elif settings_service.settings.cache_type == "async":
return AsyncInMemoryCache()

5
src/backend/base/langflow/services/database/factory.py
View file

@ -1,6 +1,5 @@
from typing import TYPE_CHECKING
from langflow.services.database.service import DatabaseService
from langflow.services.factory import ServiceFactory
@ -14,6 +13,6 @@ class DatabaseServiceFactory(ServiceFactory):
def create(self, settings_service: "SettingsService"):
# Here you would have logic to create and configure a DatabaseService
if not settings_service.settings.DATABASE_URL:
if not settings_service.settings.database_url:
raise ValueError("No database URL provided")
return DatabaseService(settings_service.settings.DATABASE_URL)
return DatabaseService(settings_service.settings.database_url)

2
src/backend/base/langflow/services/database/service.py
View file

@ -37,7 +37,7 @@ class DatabaseService(Service):
def _create_engine(self) -> "Engine":
"""Create the engine for the database."""
settings_service = get_settings_service()
if settings_service.settings.DATABASE_URL and settings_service.settings.DATABASE_URL.startswith("sqlite"):
if settings_service.settings.database_url and settings_service.settings.database_url.startswith("sqlite"):
connect_args = {"check_same_thread": False}
else:
connect_args = {}

8
src/backend/base/langflow/services/plugins/langfuse_plugin.py
View file

@ -24,12 +24,12 @@ class LangfuseInstance:
settings_manager = get_settings_service()
if settings_manager.settings.LANGFUSE_PUBLIC_KEY and settings_manager.settings.LANGFUSE_SECRET_KEY:
if settings_manager.settings.langfuse_public_key and settings_manager.settings.langfuse_secret_key:
logger.debug("Langfuse credentials found")
cls._instance = Langfuse(
public_key=settings_manager.settings.LANGFUSE_PUBLIC_KEY,
secret_key=settings_manager.settings.LANGFUSE_SECRET_KEY,
host=settings_manager.settings.LANGFUSE_HOST,
public_key=settings_manager.settings.langfuse_public_key,
secret_key=settings_manager.settings.langfuse_secret_key,
host=settings_manager.settings.langfuse_host,
)
else:
logger.debug("No Langfuse credentials found")

107
src/backend/base/langflow/services/settings/base.py
View file

@ -7,12 +7,13 @@ from typing import Any, List, Optional, Tuple, Type
import orjson
import yaml
from langflow.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
from loguru import logger
from pydantic import field_validator
from pydantic.fields import FieldInfo
from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict
from langflow.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
# BASE_COMPONENTS_PATH = str(Path(__file__).parent / "components")
BASE_COMPONENTS_PATH = str(Path(__file__).parent.parent.parent / "components")
@ -57,59 +58,43 @@ class MyCustomSource(EnvSettingsSource):
class Settings(BaseSettings):
CHAINS: dict = {}
AGENTS: dict = {}
PROMPTS: dict = {}
LLMS: dict = {}
TOOLS: dict = {}
MEMORIES: dict = {}
EMBEDDINGS: dict = {}
VECTORSTORES: dict = {}
DOCUMENTLOADERS: dict = {}
WRAPPERS: dict = {}
RETRIEVERS: dict = {}
TOOLKITS: dict = {}
TEXTSPLITTERS: dict = {}
UTILITIES: dict = {}
CUSTOM_COMPONENTS: dict = {}
# Define the default LANGFLOW_DIR
CONFIG_DIR: Optional[str] = None
config_dir: Optional[str] = None
# Define if langflow db should be saved in config dir or
# in the langflow directory
SAVE_DB_IN_CONFIG_DIR: bool = False
save_db_in_config_dir: bool = False
"""Define if langflow database should be saved in LANGFLOW_CONFIG_DIR or in the langflow directory (i.e. in the package directory)."""
DEV: bool = False
DATABASE_URL: Optional[str] = None
CACHE_TYPE: str = "async"
REMOVE_API_KEYS: bool = False
COMPONENTS_PATH: List[str] = []
LANGCHAIN_CACHE: str = "InMemoryCache"
dev: bool = False
database_url: Optional[str] = None
cache_type: str = "async"
remove_api_keys: bool = False
components_path: List[str] = []
langchain_cache: str = "InMemoryCache"
# Redis
REDIS_HOST: str = "localhost"
REDIS_PORT: int = 6379
REDIS_DB: int = 0
REDIS_URL: Optional[str] = None
REDIS_CACHE_EXPIRE: int = 3600
redis_host: str = "localhost"
redis_port: int = 6379
redis_db: int = 0
redis_url: Optional[str] = None
redis_cache_expire: int = 3600
# PLUGIN_DIR: Optional[str] = None
LANGFUSE_SECRET_KEY: Optional[str] = None
LANGFUSE_PUBLIC_KEY: Optional[str] = None
LANGFUSE_HOST: Optional[str] = None
langfuse_secret_key: Optional[str] = None
langfuse_public_key: Optional[str] = None
langfuse_host: Optional[str] = None
STORE: Optional[bool] = True
STORE_URL: Optional[str] = "https://api.langflow.store"
DOWNLOAD_WEBHOOK_URL: Optional[str] = (
store: Optional[bool] = True
store_url: Optional[str] = "https://api.langflow.store"
download_webhook_url: Optional[str] = (
"https://api.langflow.store/flows/trigger/ec611a61-8460-4438-b187-a4f65e5559d4"
)
LIKE_WEBHOOK_URL: Optional[str] = "https://api.langflow.store/flows/trigger/64275852-ec00-45c1-984e-3bff814732da"
like_webhook_url: Optional[str] = "https://api.langflow.store/flows/trigger/64275852-ec00-45c1-984e-3bff814732da"
STORAGE_TYPE: str = "local"
storage_type: str = "local"
CELERY_ENABLED: bool = False
celery_enabled: bool = False
fallback_to_env_var: bool = True
"""If set to True, Global Variables set in the UI will fallback to a environment variable
@ -120,7 +105,7 @@ class Settings(BaseSettings):
variables_to_get_from_environment: list[str] = VARIABLES_TO_GET_FROM_ENVIRONMENT
"""List of environment variables to get from the environment and store in the database."""
@field_validator("CONFIG_DIR", mode="before")
@field_validator("config_dir", mode="before")
def set_langflow_dir(cls, value):
if not value:
from platformdirs import user_cache_dir
@ -143,7 +128,7 @@ class Settings(BaseSettings):
return str(value)
@field_validator("DATABASE_URL", mode="before")
@field_validator("database_url", mode="before")
def set_database_url(cls, value, info):
if not value:
logger.debug("No database_url provided, trying LANGFLOW_DATABASE_URL env variable")
@ -151,17 +136,17 @@ class Settings(BaseSettings):
value = langflow_database_url
logger.debug("Using LANGFLOW_DATABASE_URL env variable.")
else:
logger.debug("No DATABASE_URL env variable, using sqlite database")
logger.debug("No database_url env variable, using sqlite database")
# Originally, we used sqlite:///./langflow.db
# so we need to migrate to the new format
# if there is a database in that location
if not info.data["CONFIG_DIR"]:
raise ValueError("CONFIG_DIR not set, please set it or provide a DATABASE_URL")
if not info.data["config_dir"]:
raise ValueError("config_dir not set, please set it or provide a database_url")
from langflow.version import is_pre_release # type: ignore
if info.data["SAVE_DB_IN_CONFIG_DIR"]:
database_dir = info.data["CONFIG_DIR"]
logger.debug(f"Saving database to CONFIG_DIR: {database_dir}")
if info.data["save_db_in_config_dir"]:
database_dir = info.data["config_dir"]
logger.debug(f"Saving database to config_dir: {database_dir}")
else:
database_dir = Path(__file__).parent.parent.parent.resolve()
logger.debug(f"Saving database to langflow directory: {database_dir}")
@ -174,12 +159,12 @@ class Settings(BaseSettings):
if is_pre_release:
if Path(new_pre_path).exists():
final_path = new_pre_path
elif Path(new_path).exists() and info.data["SAVE_DB_IN_CONFIG_DIR"]:
elif Path(new_path).exists() and info.data["save_db_in_config_dir"]:
# We need to copy the current db to the new location
logger.debug("Copying existing database to new location")
copy2(new_path, new_pre_path)
logger.debug(f"Copied existing database to {new_pre_path}")
elif Path(f"./{db_file_name}").exists() and info.data["SAVE_DB_IN_CONFIG_DIR"]:
elif Path(f"./{db_file_name}").exists() and info.data["save_db_in_config_dir"]:
logger.debug("Copying existing database to new location")
copy2(f"./{db_file_name}", new_pre_path)
logger.debug(f"Copied existing database to {new_pre_path}")
@ -211,7 +196,7 @@ class Settings(BaseSettings):
return value
@field_validator("COMPONENTS_PATH", mode="before")
@field_validator("components_path", mode="before")
def set_components_path(cls, value):
if os.getenv("LANGFLOW_COMPONENTS_PATH"):
logger.debug("Adding LANGFLOW_COMPONENTS_PATH to components_path")
@ -240,23 +225,8 @@ class Settings(BaseSettings):
def update_from_yaml(self, file_path: str, dev: bool = False):
new_settings = load_settings_from_yaml(file_path)
self.CHAINS = new_settings.CHAINS or {}
self.AGENTS = new_settings.AGENTS or {}
self.PROMPTS = new_settings.PROMPTS or {}
self.LLMS = new_settings.LLMS or {}
self.TOOLS = new_settings.TOOLS or {}
self.MEMORIES = new_settings.MEMORIES or {}
self.WRAPPERS = new_settings.WRAPPERS or {}
self.TOOLKITS = new_settings.TOOLKITS or {}
self.TEXTSPLITTERS = new_settings.TEXTSPLITTERS or {}
self.UTILITIES = new_settings.UTILITIES or {}
self.EMBEDDINGS = new_settings.EMBEDDINGS or {}
self.VECTORSTORES = new_settings.VECTORSTORES or {}
self.DOCUMENTLOADERS = new_settings.DOCUMENTLOADERS or {}
self.RETRIEVERS = new_settings.RETRIEVERS or {}
self.CUSTOM_COMPONENTS = new_settings.CUSTOM_COMPONENTS or {}
self.COMPONENTS_PATH = new_settings.COMPONENTS_PATH or []
self.DEV = dev
self.components_path = new_settings.components_path or []
self.dev = dev
def update_settings(self, **kwargs):
logger.debug("Updating settings")
@ -325,6 +295,3 @@ def load_settings_from_yaml(file_path: str) -> Settings:
logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}")
return Settings(**settings_dict)
return Settings(**settings_dict)
return Settings(**settings_dict)
return Settings(**settings_dict)

4
src/backend/base/langflow/services/settings/manager.py
View file

@ -35,10 +35,10 @@ class SettingsService(Service):
logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}")
settings = Settings(**settings_dict)
if not settings.CONFIG_DIR:
if not settings.config_dir:
raise ValueError("CONFIG_DIR must be set in settings")
auth_settings = AuthSettings(
CONFIG_DIR=settings.CONFIG_DIR,
CONFIG_DIR=settings.config_dir,
)
return cls(settings, auth_settings)

7
src/backend/base/langflow/services/settings/service.py
View file

@ -1,10 +1,11 @@
import os
import yaml
from loguru import logger
from langflow.services.base import Service
from langflow.services.settings.auth import AuthSettings
from langflow.services.settings.base import Settings
from loguru import logger
class SettingsService(Service):
@ -34,10 +35,10 @@ class SettingsService(Service):
logger.debug(f"Loading {len(settings_dict[key])} {key} from {file_path}")
settings = Settings(**settings_dict)
if not settings.CONFIG_DIR:
if not settings.config_dir:
raise ValueError("CONFIG_DIR must be set in settings")
auth_settings = AuthSettings(
CONFIG_DIR=settings.CONFIG_DIR,
CONFIG_DIR=settings.config_dir,
)
return cls(settings, auth_settings)

2
src/backend/base/langflow/services/storage/factory.py
View file

@ -13,7 +13,7 @@ class StorageServiceFactory(ServiceFactory):
)
def create(self, session_service: SessionService, settings_service: SettingsService):
storage_type = settings_service.settings.STORAGE_TYPE
storage_type = settings_service.settings.storage_type
if storage_type.lower() == "local":
from .local import LocalStorageService

2
src/backend/base/langflow/services/storage/local.py
View file

@ -11,7 +11,7 @@ class LocalStorageService(StorageService):
def __init__(self, session_service, settings_service):
"""Initialize the local storage service with session and settings services."""
super().__init__(session_service, settings_service)
self.data_dir = Path(settings_service.settings.CONFIG_DIR)
self.data_dir = Path(settings_service.settings.config_dir)
self.set_ready()
def build_full_path(self, flow_id: str, file_name: str) -> str:

6
src/backend/base/langflow/services/store/service.py
View file

@ -79,9 +79,9 @@ class StoreService(Service):
def __init__(self, settings_service: "SettingsService"):
self.settings_service = settings_service
self.base_url = self.settings_service.settings.STORE_URL
self.download_webhook_url = self.settings_service.settings.DOWNLOAD_WEBHOOK_URL
self.like_webhook_url = self.settings_service.settings.LIKE_WEBHOOK_URL
self.base_url = self.settings_service.settings.store_url
self.download_webhook_url = self.settings_service.settings.download_webhook_url
self.like_webhook_url = self.settings_service.settings.like_webhook_url
self.components_url = f"{self.base_url}/items/components"
self.default_fields = [
"id",

2
src/backend/base/langflow/services/task/service.py
View file

@ -29,7 +29,7 @@ class TaskService(Service):
def __init__(self, settings_service: "SettingsService"):
self.settings_service = settings_service
try:
if self.settings_service.settings.CELERY_ENABLED:
if self.settings_service.settings.celery_enabled:
USE_CELERY = True
status = check_celery_availability()

10
src/backend/base/langflow/utils/util.py
View file

@ -6,6 +6,8 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from docstring_parser import parse
from langflow.schema.schema import Record
from langflow.services.deps import get_settings_service
from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
@ -438,13 +440,13 @@ def update_settings(
settings_service.settings.update_from_yaml(config, dev=dev)
if remove_api_keys:
logger.debug(f"Setting remove_api_keys to {remove_api_keys}")
settings_service.settings.update_settings(REMOVE_API_KEYS=remove_api_keys)
settings_service.settings.update_settings(remove_api_keys=remove_api_keys)
if cache:
logger.debug(f"Setting cache to {cache}")
settings_service.settings.update_settings(CACHE=cache)
settings_service.settings.update_settings(cache=cache)
if components_path:
logger.debug(f"Adding component path {components_path}")
settings_service.settings.update_settings(COMPONENTS_PATH=components_path)
settings_service.settings.update_settings(components_path=components_path)
if not store:
logger.debug("Setting store to False")
settings_service.settings.update_settings(STORE=False)
settings_service.settings.update_settings(store=False)

3
tests/test_cli.py
View file

@ -2,7 +2,6 @@ from pathlib import Path
from tempfile import tempdir
import pytest
from langflow.__main__ import app
from langflow.services import deps
@ -29,7 +28,7 @@ def test_components_path(runner, client, default_settings):
)
assert result.exit_code == 0, result.stdout
settings_service = deps.get_settings_service()
assert str(temp_dir) in settings_service.settings.COMPONENTS_PATH
assert str(temp_dir) in settings_service.settings.components_path
def test_superuser(runner, client, session):

3
tests/test_endpoints.py
View file

@ -4,6 +4,7 @@ from uuid import UUID, uuid4
import pytest
from fastapi import status
from fastapi.testclient import TestClient
from langflow.custom.directory_reader.directory_reader import DirectoryReader
from langflow.services.deps import get_settings_service
@ -263,7 +264,7 @@ def test_get_all(client: TestClient, logged_in_headers):
response = client.get("api/v1/all", headers=logged_in_headers)
assert response.status_code == 200
settings = get_settings_service().settings
dir_reader = DirectoryReader(settings.COMPONENTS_PATH[0])
dir_reader = DirectoryReader(settings.components_path[0])
files = dir_reader.get_files()
# json_response is a dict of dicts
all_names = [component_name for _, components in response.json().items() for component_name in components]