Add environment variable initialization and store_environment_variables (#1654)

* Add environment variable initialization and add store_environment_variables

* Add variables_to_get_from_environment to store specific environment variables

* Remove unused variables from VariableService

* Update global variables documentation and refactor VariableService

.env.example

@@ -71,6 +71,10 @@ LANGFLOW_SUPERUSER=
 # Example: LANGFLOW_SUPERUSER_PASSWORD=123456
 LANGFLOW_SUPERUSER_PASSWORD=
 
+# Should store environment variables in the database
+# Values: true, false
+LANGFLOW_STORE_ENVIRONMENT_VARIABLES=
+
 # STORE_URL
 # Example: LANGFLOW_STORE_URL=https://api.langflow.store
 # LANGFLOW_STORE_URL=

docs/docs/migration/global-variables.mdx

@@ -3,6 +3,15 @@ import Admonition from "@theme/Admonition";
 
 # Global Variables
 
+## TLDR;
+
+- Global Variables are reusable variables that can be accessed from any Text field in your project.
+- To create a Global Variable, click on the 🌐 button in a Text field and then **+ Add New Variable**.
+- Define the **Name**, **Type**, and **Value** of the variable.
+- Click on **Save Variable** to create the variable.
+- All Credential Global Variables are encrypted and cannot be accessed by anyone but you.
+- Set _`LANGFLOW_STORE_ENVIRONMENT_VARIABLES`_ to _`true`_ in your `.env` file to add all variables in _`LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT`_ to your user's Global Variables.
+
 Global Variables are a really useful feature of Langflow.
 They allow you to define reusable variables that can be accessed from any Text field in your project.
 
@@ -48,7 +57,8 @@ The **Value** is the value that the variable will have.
 {/* say that all variables are encrypted */}
 
 <Admonition type="warning">
-  All Global Variables are encrypted and cannot be accessed by anyone but you.
+  All Credential Global Variables are encrypted and cannot be accessed by anyone
+  but you.
 </Admonition>
 
 <ZoomableImage
@@ -63,3 +73,37 @@ The **Value** is the value that the variable will have.
 After you have defined your variable, click on **Save Variable** and your variable will be created.
 
 After that, once you click on the 🌐 button in a Text field, you will see your new variable in the dropdown.
+
+## Environment Variables
+
+If you set _`LANGFLOW_STORE_ENVIRONMENT_VARIABLES`_ to _`true`_ (which is the default value) in your `.env` file, all variables in _`LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT`_ will be added to your user's Global Variables.
+
+All of these variables can be used in your project as any other Global Variable.
+
+<Admonition type="tip">
+  You can set _`LANGFLOW_STORE_ENVIRONMENT_VARIABLES`_ to _`false`_ in your
+  `.env` file to prevent this behavior.
+</Admonition>
+
+You can also set _`LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT`_ to a list of variables that you want to get from the environment.
+
+The default list at the moment is:
+
+- ANTHROPIC_API_KEY
+- ASTRA_DB_API_ENDPOINT
+- ASTRA_DB_APPLICATION_TOKEN
+- AZURE_OPENAI_API_KEY
+- AZURE_OPENAI_API_DEPLOYMENT_NAME
+- AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME
+- AZURE_OPENAI_API_INSTANCE_NAME
+- AZURE_OPENAI_API_VERSION
+- COHERE_API_KEY
+- GOOGLE_API_KEY
+- HUGGINGFACEHUB_API_TOKEN
+- OPENAI_API_KEY
+
+<Admonition type="tip">
+  Set _`LANGFLOW_VARIABLES_TO_GET_FROM_ENVIRONMENT`_ as a comma-separated list
+  of variables (e.g. _`"VARIABLE1, VARIABLE2"`_) or as a JSON-encoded string
+  (e.g. _`'["VARIABLE1", "VARIABLE2"]'`_).
+</Admonition>

src/backend/base/langflow/api/v1/login.py

@@ -1,7 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
 from fastapi.security import OAuth2PasswordRequestForm
-from sqlmodel import Session
-
 from langflow.api.v1.schemas import Token
 from langflow.services.auth.utils import (
     authenticate_user,
@@ -9,8 +7,14 @@ from langflow.services.auth.utils import (
     create_user_longterm_token,
     create_user_tokens,
 )
-from langflow.services.deps import get_session, get_settings_service
+from langflow.services.deps import (
+    get_session,
+    get_settings_service,
+    get_variable_service,
+)
 from langflow.services.settings.manager import SettingsService
+from langflow.services.variable.service import VariableService
+from sqlmodel import Session
 
 router = APIRouter(tags=["Login"])
 
@@ -22,6 +26,7 @@ async def login_to_get_access_token(
     db: Session = Depends(get_session),
     # _: Session = Depends(get_current_active_user)
     settings_service=Depends(get_settings_service),
+    variable_service: VariableService = Depends(get_variable_service),
 ):
     auth_settings = settings_service.auth_settings
     try:
@@ -52,6 +57,7 @@ async def login_to_get_access_token(
             secure=auth_settings.ACCESS_SECURE,
             expires=auth_settings.ACCESS_TOKEN_EXPIRE_SECONDS,
         )
+        variable_service.initialize_user_variables(user.id, db)
         return tokens
     else:
         raise HTTPException(
@@ -66,10 +72,11 @@ async def auto_login(
     response: Response,
     db: Session = Depends(get_session),
     settings_service=Depends(get_settings_service),
+    variable_service: VariableService = Depends(get_variable_service),
 ):
     auth_settings = settings_service.auth_settings
     if settings_service.auth_settings.AUTO_LOGIN:
-        tokens = create_user_longterm_token(db)
+        user_id, tokens = create_user_longterm_token(db)
         response.set_cookie(
             "access_token_lf",
             tokens["access_token"],
@@ -78,6 +85,7 @@ async def auto_login(
             secure=auth_settings.ACCESS_SECURE,
             expires=None,  # Set to None to make it a session cookie
         )
+        variable_service.initialize_user_variables(user_id, db)
         return tokens
 
     raise HTTPException(
@@ -91,7 +99,9 @@ async def auto_login(
 
 @router.post("/refresh")
 async def refresh_token(
-    request: Request, response: Response, settings_service: "SettingsService" = Depends(get_settings_service)
+    request: Request,
+    response: Response,
+    settings_service: "SettingsService" = Depends(get_settings_service),
 ):
     auth_settings = settings_service.auth_settings
 
@@ -129,3 +139,4 @@ async def logout(response: Response):
     response.delete_cookie("refresh_token_lf")
     response.delete_cookie("access_token_lf")
     return {"message": "Logout successful"}
+    return {"message": "Logout successful"}

src/backend/base/langflow/services/auth/utils.py

@@ -11,7 +11,11 @@ from starlette.websockets import WebSocket
 
 from langflow.services.database.models.api_key.crud import check_key
 from langflow.services.database.models.api_key.model import ApiKey
-from langflow.services.database.models.user.crud import get_user_by_id, get_user_by_username, update_user_last_login_at
+from langflow.services.database.models.user.crud import (
+    get_user_by_id,
+    get_user_by_username,
+    update_user_last_login_at,
+)
 from langflow.services.database.models.user.model import User
 from langflow.services.deps import get_session, get_settings_service
 
@@ -227,7 +231,7 @@ def create_user_longterm_token(db: Session = Depends(get_session)) -> dict:
     # Update: last_login_at
     update_user_last_login_at(super_user.id, db)
 
-    return {
+    return super_user.id, {
         "access_token": access_token,
         "refresh_token": None,
         "token_type": "bearer",

src/backend/base/langflow/services/settings/base.py

@@ -3,18 +3,51 @@ import json
 import os
 from pathlib import Path
 from shutil import copy2
-from typing import List, Optional
+from typing import Any, List, Optional, Tuple, Type
 
 import orjson
 import yaml
 from loguru import logger
 from pydantic import field_validator, validator
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic.fields import FieldInfo
+from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict
+
+from langflow.services.settings.constants import VARIABLES_TO_GET_FROM_ENVIRONMENT
+from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict
 
 # BASE_COMPONENTS_PATH = str(Path(__file__).parent / "components")
 BASE_COMPONENTS_PATH = str(Path(__file__).parent.parent.parent / "components")
 
 
+def is_list_of_any(field: FieldInfo) -> bool:
+    """
+    Check if the given field is a list or an optional list of any type.
+
+    Args:
+        field (FieldInfo): The field to be checked.
+
+    Returns:
+        bool: True if the field is a list or a list of any type, False otherwise.
+    """
+    return field.annotation.__origin__ == list or any(
+        arg.__origin__ == list for arg in field.annotation.__args__ if hasattr(arg, "__origin__")
+    )
+
+
+class MyCustomSource(EnvSettingsSource):
+    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
+        # allow comma-separated list parsing
+
+        # fieldInfo contains the annotation of the field
+        if is_list_of_any(field):
+            if isinstance(value, str):
+                value = value.split(",")
+            if isinstance(value, list):
+                return value
+
+        return super().prepare_field_value(field_name, field, value, value_is_complex)
+
+
 class Settings(BaseSettings):
     CHAINS: dict = {}
     AGENTS: dict = {}
@@ -67,6 +100,11 @@ class Settings(BaseSettings):
 
     CELERY_ENABLED: bool = False
 
+    store_environment_variables: bool = True
+    """Whether to store environment variables as Global Variables in the database."""
+    variables_to_get_from_environment: list[str] = VARIABLES_TO_GET_FROM_ENVIRONMENT
+    """List of environment variables to get from the environment and store in the database."""
+
     @validator("CONFIG_DIR", pre=True, allow_reuse=True)
     def set_langflow_dir(cls, value):
         if not value:
@@ -149,14 +187,6 @@ class Settings(BaseSettings):
 
     model_config = SettingsConfigDict(validate_assignment=True, extra="ignore", env_prefix="LANGFLOW_")
 
-    # @model_validator()
-    # @classmethod
-    # def validate_lists(cls, values):
-    #     for key, value in values.items():
-    #         if key != "dev" and not value:
-    #             values[key] = []
-    #     return values
-
     def update_from_yaml(self, file_path: str, dev: bool = False):
         new_settings = load_settings_from_yaml(file_path)
         self.CHAINS = new_settings.CHAINS or {}
@@ -209,6 +239,17 @@ class Settings(BaseSettings):
                 logger.debug(f"Updated {key}")
             logger.debug(f"{key}: {getattr(self, key)}")
 
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        return (MyCustomSource(settings_cls),)
+
 
 def save_settings_to_yaml(settings: Settings, file_path: str):
     with open(file_path, "w") as f:

src/backend/base/langflow/services/settings/constants.py

@@ -1,2 +1,16 @@
 DEFAULT_SUPERUSER = "langflow"
 DEFAULT_SUPERUSER_PASSWORD = "langflow"
+VARIABLES_TO_GET_FROM_ENVIRONMENT = [
+    "OPENAI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "GOOGLE_API_KEY",
+    "AZURE_OPENAI_API_KEY",
+    "AZURE_OPENAI_API_VERSION",
+    "AZURE_OPENAI_API_INSTANCE_NAME",
+    "AZURE_OPENAI_API_DEPLOYMENT_NAME",
+    "AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME",
+    "ASTRA_DB_APPLICATION_TOKEN",
+    "ASTRA_DB_API_ENDPOINT",
+    "COHERE_API_KEY",
+    "HUGGINGFACEHUB_API_TOKEN",
+]

src/backend/base/langflow/services/variable/service.py

@@ -1,13 +1,14 @@
+import os
 from typing import TYPE_CHECKING, Optional, Union
 from uuid import UUID
 
 from fastapi import Depends
-from sqlmodel import Session, select
-
 from langflow.services.auth import utils as auth_utils
 from langflow.services.base import Service
 from langflow.services.database.models.variable.model import Variable
 from langflow.services.deps import get_session
+from loguru import logger
+from sqlmodel import Session, select
 
 if TYPE_CHECKING:
     from langflow.services.settings.service import SettingsService
@@ -19,7 +20,28 @@ class VariableService(Service):
     def __init__(self, settings_service: "SettingsService"):
         self.settings_service = settings_service
 
-    def get_variable(self, user_id: Union[UUID, str], name: str, session: Session = Depends(get_session)) -> str:
+    def initialize_user_variables(self, user_id: Union[UUID, str], session: Session = Depends(get_session)):
+        # Check for environment variables that should be stored in the database
+        should_or_should_not = "Should" if self.settings_service.settings.store_environment_variables else "Should not"
+        logger.info(f"{should_or_should_not} store environment variables in the database.")
+        if self.settings_service.settings.store_environment_variables:
+            for var in self.settings_service.settings.variables_to_get_from_environment:
+                if var in os.environ:
+                    logger.debug(f"Creating {var} variable from environment.")
+                    try:
+                        self.create_variable(user_id, var, os.environ[var], _type="Credential", session=session)
+                    except Exception as e:
+                        logger.error(f"Error creating {var} variable: {e}")
+
+        else:
+            logger.info("Skipping environment variable storage.")
+
+    def get_variable(
+        self,
+        user_id: Union[UUID, str],
+        name: str,
+        session: Session = Depends(get_session),
+    ) -> str:
         # we get the credential from the database
         # credential = session.query(Variable).filter(Variable.user_id == user_id, Variable.name == name).first()
         variable = session.exec(select(Variable).where(Variable.user_id == user_id, Variable.name == name)).first()
@@ -34,7 +56,11 @@ class VariableService(Service):
         return [variable.name for variable in variables]
 
     def update_variable(
-        self, user_id: Union[UUID, str], name: str, value: str, session: Session = Depends(get_session)
+        self,
+        user_id: Union[UUID, str],
+        name: str,
+        value: str,
+        session: Session = Depends(get_session),
     ):
         variable = session.exec(select(Variable).where(Variable.user_id == user_id, Variable.name == name)).first()
         if not variable:
@@ -46,7 +72,12 @@ class VariableService(Service):
         session.refresh(variable)
         return variable
 
-    def delete_variable(self, user_id: Union[UUID, str], name: str, session: Session = Depends(get_session)):
+    def delete_variable(
+        self,
+        user_id: Union[UUID, str],
+        name: str,
+        session: Session = Depends(get_session),
+    ):
         variable = session.exec(select(Variable).where(Variable.user_id == user_id, Variable.name == name)).first()
         if not variable:
             raise ValueError(f"{name} variable not found.")
@@ -55,12 +86,21 @@ class VariableService(Service):
         return variable
 
     def create_variable(
-        self, user_id: Union[UUID, str], name: str, value: str, session: Session = Depends(get_session)
+        self,
+        user_id: Union[UUID, str],
+        name: str,
+        value: str,
+        _type: str = "Generic",
+        session: Session = Depends(get_session),
     ):
         variable = Variable(
-            user_id=user_id, name=name, value=auth_utils.encrypt_api_key(value, settings_service=self.settings_service)
+            user_id=user_id,
+            name=name,
+            type=_type,
+            value=auth_utils.encrypt_api_key(value, settings_service=self.settings_service),
         )
         session.add(variable)
         session.commit()
         session.refresh(variable)
         return variable
+        return variable