feat: implement function to truncate json (#5811)

* feat: implement function to truncate json

* refactor: apply JSON truncation function in models

* fix: update code to use the standard field serialization format

* Update src/backend/base/langflow/services/database/models/transactions/model.py

Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>

* Update src/backend/base/langflow/services/database/models/transactions/model.py

Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>

* Update src/backend/base/langflow/services/database/models/vertex_builds/model.py

---------

Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>

src/backend/base/langflow/services/database/models/transactions/model.py

@@ -5,11 +5,11 @@ from uuid import UUID, uuid4
 from pydantic import field_serializer, field_validator
 from sqlmodel import JSON, Column, Field, Relationship, SQLModel
 
+from langflow.services.database.utils import truncate_json
+
 if TYPE_CHECKING:
     from langflow.services.database.models.flow.model import Flow
 
-from langflow.utils.util_strings import truncate_long_strings
-
 
 class TransactionBase(SQLModel):
     timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
@@ -34,9 +34,13 @@ class TransactionBase(SQLModel):
             value = UUID(value)
         return value
 
+    @field_serializer("inputs")
+    def serialize_inputs(self, data) -> dict:
+        return truncate_json(data)
+
     @field_serializer("outputs")
     def serialize_outputs(self, data) -> dict:
-        return truncate_long_strings(data)
+        return truncate_json(data)
 
 
 class TransactionTable(TransactionBase, table=True):  # type: ignore[call-arg]

src/backend/base/langflow/services/database/models/vertex_builds/model.py

@@ -6,6 +6,8 @@ from pydantic import BaseModel, field_serializer, field_validator
 from sqlalchemy import Text
 from sqlmodel import JSON, Column, Field, Relationship, SQLModel
 
+from langflow.services.database.utils import truncate_json
+
 if TYPE_CHECKING:
     from langflow.services.database.models.flow.model import Flow
 
@@ -42,12 +44,12 @@ class VertexBuildBase(SQLModel):
         return value
 
     @field_serializer("data")
-    def serialize_data(self, data: dict) -> dict:
-        return truncate_long_strings(data)
+    def serialize_data(self, data) -> dict:
+        return truncate_json(data)
 
     @field_serializer("artifacts")
     def serialize_artifacts(self, data) -> dict:
-        return truncate_long_strings(data)
+        return truncate_json(data)
 
     @field_serializer("params")
     def serialize_params(self, data) -> str:

src/backend/base/langflow/services/database/utils.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
@@ -9,10 +10,54 @@ from loguru import logger
 from sqlmodel import text
 from sqlmodel.ext.asyncio.session import AsyncSession
 
+from langflow.utils import constants
+
 if TYPE_CHECKING:
     from langflow.services.database.service import DatabaseService
 
 
+def truncate_json(data, *, max_size: int = constants.MAX_TEXT_LENGTH):
+    def calculate_size(data):
+        return len(json.dumps(data))
+
+    def shrink_to_size(data, remaining_size):
+        if isinstance(data, dict):
+            truncated = {}
+            for key, value in data.items():
+                key_size = len(json.dumps(key))
+                if remaining_size - key_size <= 0:
+                    break
+                truncated[key] = shrink_to_size(value, remaining_size - key_size)
+                remaining_size -= len(json.dumps({key: value})) - key_size
+            return truncated
+
+        if isinstance(data, list):
+            truncated = []
+            for item in data:
+                if remaining_size <= len('""'):
+                    break
+                truncated.append(shrink_to_size(item, remaining_size))
+                remaining_size -= len(json.dumps(item)) + 1
+            return truncated
+
+        if isinstance(data, str):
+            max_string_length = max(remaining_size - 2, 0)
+            return data[:max_string_length] + "…" if max_string_length > 0 else "…"
+
+        return data
+
+    try:
+        json.dumps(data)
+        is_serialized = True
+    except Exception:  # noqa: BLE001
+        is_serialized = False
+
+    if calculate_size(data) <= max_size or not is_serialized:
+        return data
+
+    return shrink_to_size(data, max_size)
+
+
 async def initialize_database(*, fix_migration: bool = False) -> None:
     logger.debug("Initializing database")
     from langflow.services.deps import get_db_service