fix: remove use of ImagePromptTemplate in image handling and adds image utils (#4467)

* remove poetry.lock

* fix: update langchain-core dependency version to 0.3.15

* feat: add functions to convert images to base64 and create data URLs

* refactor: simplify image URL handling by replacing ImagePromptTemplate with create_data_url function

* Fix image URL structure in data schema to use nested dictionary format

* Add unit tests for Data schema message conversion with text and images

* test: add unit tests for image utility functions to validate base64 conversion and data URL creation

* Refactor image URL generation to use `create_data_url` utility function instead of `ImagePromptTemplate`

* Add unit tests for message handling and image processing in schema module

- Introduce fixtures for temporary cache directory and sample image creation.
- Add tests for message creation from human and AI text.
- Implement tests for messages with single and multiple images.
- Include tests for invalid image paths and messages without sender.
- Add message serialization and conversion tests.
- Ensure cleanup of cache directory after tests.

* Use platformdirs to determine cache directory paths in test_schema_message.py

src/backend/tests/unit/schema/test_schema_data.py

@@ -0,0 +1,92 @@
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
+from langflow.schema.data import Data
+from langflow.utils.constants import MESSAGE_SENDER_AI, MESSAGE_SENDER_USER
+
+
+@pytest.fixture
+def sample_image(tmp_path):
+    """Create a sample image file for testing."""
+    image_path = tmp_path / "test_image.png"
+    # Create a small black 1x1 pixel PNG file
+    import base64
+
+    image_content = base64.b64decode(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="
+    )
+    image_path.write_bytes(image_content)
+    return image_path
+
+
+class TestDataSchema:
+    def test_data_to_message_with_text_only(self):
+        """Test conversion of Data to Message with text only."""
+        data = Data(data={"text": "Hello, world!", "sender": MESSAGE_SENDER_USER})
+        message = data.to_lc_message()
+        assert isinstance(message, HumanMessage)
+        assert message.content == [{"type": "text", "text": "Hello, world!"}]
+
+    def test_data_to_message_with_image(self, sample_image):
+        """Test conversion of Data to Message with text and image."""
+        data = Data(data={"text": "Check out this image", "sender": MESSAGE_SENDER_USER, "files": [str(sample_image)]})
+        message = data.to_lc_message()
+
+        assert isinstance(message, HumanMessage)
+        assert isinstance(message.content, list)
+        assert len(message.content) == 2
+
+        # Check text content
+        assert message.content[0] == {"type": "text", "text": "Check out this image"}
+
+        # Check image content
+        assert message.content[1]["type"] == "image_url"
+        assert "url" in message.content[1]["image_url"]
+        assert message.content[1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+    def test_data_to_message_with_multiple_images(self, sample_image, tmp_path):
+        """Test conversion of Data to Message with multiple images."""
+        # Create a second image
+        second_image = tmp_path / "second_image.png"
+        second_image.write_bytes(sample_image.read_bytes())
+
+        data = Data(
+            data={
+                "text": "Multiple images",
+                "sender": MESSAGE_SENDER_USER,
+                "files": [str(sample_image), str(second_image)],
+            }
+        )
+        message = data.to_lc_message()
+
+        assert isinstance(message, HumanMessage)
+        assert isinstance(message.content, list)
+        assert len(message.content) == 3  # text + 2 images
+
+        # Check text content
+        assert message.content[0]["type"] == "text"
+
+        # Check both images
+        assert message.content[1]["type"] == "image_url"
+        assert message.content[2]["type"] == "image_url"
+        assert all(content["image_url"]["url"].startswith("data:image/png;base64,") for content in message.content[1:])
+
+    def test_data_to_message_ai_response(self):
+        """Test conversion of Data to AI Message."""
+        data = Data(data={"text": "AI response", "sender": MESSAGE_SENDER_AI})
+        message = data.to_lc_message()
+        assert isinstance(message, AIMessage)
+        assert message.content == "AI response"
+
+    def test_data_to_message_missing_required_keys(self):
+        """Test conversion fails with missing required keys."""
+        data = Data(data={"incomplete": "data"})
+        with pytest.raises(ValueError, match="Missing required keys"):
+            data.to_lc_message()
+
+    def test_data_to_message_invalid_image_path(self, tmp_path):
+        """Test handling of invalid image path."""
+        non_existent_image = tmp_path / "non_existent.png"
+        data = Data(data={"text": "Invalid image", "sender": MESSAGE_SENDER_USER, "files": [str(non_existent_image)]})
+
+        with pytest.raises(FileNotFoundError):
+            data.to_lc_message()

src/backend/tests/unit/schema/test_schema_message.py

@@ -1,5 +1,50 @@
+import shutil
+from pathlib import Path
+
+import pytest
+from langchain_core.messages import AIMessage, HumanMessage
 from langchain_core.prompts.chat import ChatPromptTemplate
 from langflow.schema.message import Message
+from langflow.utils.constants import MESSAGE_SENDER_AI, MESSAGE_SENDER_USER
+from platformdirs import user_cache_dir
+
+
+@pytest.fixture
+def langflow_cache_dir(tmp_path):
+    """Create a temporary langflow cache directory."""
+    cache_dir = tmp_path / "langflow"
+    cache_dir.mkdir(parents=True)
+    return cache_dir
+
+
+@pytest.fixture
+def sample_image(langflow_cache_dir):
+    """Create a sample image file for testing."""
+    # Create the test_flow directory in the cache
+    flow_dir = langflow_cache_dir / "test_flow"
+    flow_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create the image in the flow directory
+    image_path = flow_dir / "test_image.png"
+    # Create a small black 1x1 pixel PNG file
+    import base64
+
+    image_content = base64.b64decode(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="
+    )
+    image_path.write_bytes(image_content)
+
+    # Use platformdirs to get the cache directory
+    real_cache_dir = Path(user_cache_dir("langflow"))
+    real_cache_dir.mkdir(parents=True, exist_ok=True)
+    real_flow_dir = real_cache_dir / "test_flow"
+    real_flow_dir.mkdir(parents=True, exist_ok=True)
+
+    # Copy the image to the real cache location
+    real_image_path = real_flow_dir / "test_image.png"
+    shutil.copy2(str(image_path), str(real_image_path))
+
+    return image_path
 
 
 def test_message_prompt_serialization():
@@ -10,3 +55,125 @@ def test_message_prompt_serialization():
     prompt = message.load_lc_prompt()
     assert isinstance(prompt, ChatPromptTemplate)
     assert prompt.messages[0].content == "Hello, Langflow!"
+
+
+def test_message_from_human_text():
+    """Test creating a message from human text."""
+    text = "Hello, AI!"
+    message = Message(text=text, sender=MESSAGE_SENDER_USER)
+    lc_message = message.to_lc_message()
+
+    assert isinstance(lc_message, HumanMessage)
+    assert lc_message.content == text
+
+
+def test_message_from_ai_text():
+    """Test creating a message from AI text."""
+    text = "Hello, Human!"
+    message = Message(text=text, sender=MESSAGE_SENDER_AI)
+    lc_message = message.to_lc_message()
+
+    assert isinstance(lc_message, AIMessage)
+    assert lc_message.content == text
+
+
+def test_message_with_single_image(sample_image):
+    """Test creating a message with text and an image."""
+    text = "Check out this image"
+    # Format the file path as expected: "flow_id/filename"
+    file_path = f"test_flow/{sample_image.name}"
+    message = Message(text=text, sender=MESSAGE_SENDER_USER, files=[file_path])
+    lc_message = message.to_lc_message()
+
+    assert isinstance(lc_message, HumanMessage)
+    assert isinstance(lc_message.content, list)
+    assert len(lc_message.content) == 2
+
+    # Check text content
+    assert lc_message.content[0] == {"type": "text", "text": text}
+
+    # Check image content
+    assert lc_message.content[1]["type"] == "image_url"
+    assert "url" in lc_message.content[1]["image_url"]
+    assert lc_message.content[1]["image_url"]["url"].startswith("data:image/png;base64,")
+
+
+def test_message_with_multiple_images(sample_image, langflow_cache_dir):
+    """Test creating a message with multiple images."""
+    # Create a second image in the cache directory
+    flow_dir = langflow_cache_dir / "test_flow"
+    second_image = flow_dir / "second_image.png"
+    shutil.copy2(str(sample_image), str(second_image))
+
+    # Use platformdirs for the real cache location
+    real_cache_dir = Path(user_cache_dir("langflow")) / "test_flow"
+    real_cache_dir.mkdir(parents=True, exist_ok=True)
+    real_second_image = real_cache_dir / "second_image.png"
+    shutil.copy2(str(sample_image), str(real_second_image))
+
+    text = "Multiple images"
+    message = Message(
+        text=text,
+        sender=MESSAGE_SENDER_USER,
+        files=[f"test_flow/{sample_image.name}", f"test_flow/{second_image.name}"],
+    )
+    lc_message = message.to_lc_message()
+
+    assert isinstance(lc_message, HumanMessage)
+    assert isinstance(lc_message.content, list)
+    assert len(lc_message.content) == 3  # text + 2 images
+
+    # Check text content
+    assert lc_message.content[0] == {"type": "text", "text": text}
+
+    # Check both images
+    assert all(
+        content["type"] == "image_url" and content["image_url"]["url"].startswith("data:image/png;base64,")
+        for content in lc_message.content[1:]
+    )
+
+
+def test_message_with_invalid_image_path():
+    """Test handling of invalid image path."""
+    file_path = "test_flow/non_existent.png"
+    message = Message(text="Invalid image", sender=MESSAGE_SENDER_USER, files=[file_path])
+
+    with pytest.raises(FileNotFoundError):
+        message.to_lc_message()
+
+
+def test_message_without_sender():
+    """Test message creation without sender specification."""
+    # Create message without sender
+    message = Message(text="Test message")
+    # Verify the message was created but has no sender
+    assert message.text == "Test message"
+    assert message.sender is None
+
+
+def test_message_serialization():
+    """Test message serialization to dict."""
+    message = Message(text="Test message", sender=MESSAGE_SENDER_USER)
+    serialized = message.model_dump()
+
+    assert serialized["text"] == "Test message"
+    assert serialized["sender"] == MESSAGE_SENDER_USER
+
+
+def test_message_to_lc_without_sender():
+    """Test converting a message without sender to langchain message."""
+    message = Message(text="Test message")
+    # When no sender is specified, it defaults to HumanMessage
+    lc_message = message.to_lc_message()
+    assert isinstance(lc_message, HumanMessage)
+    assert lc_message.content == "Test message"
+
+
+# Clean up the cache directory after all tests
+@pytest.fixture(autouse=True)
+def cleanup():
+    yield
+    # Clean up the real cache directory after tests
+    cache_dir = Path(user_cache_dir("langflow"))
+    if cache_dir.exists():
+        shutil.rmtree(str(cache_dir))

src/backend/tests/unit/utils/test_image_utils.py

@@ -0,0 +1,66 @@
+import base64
+
+import pytest
+from langflow.utils.image import convert_image_to_base64, create_data_url
+
+
+@pytest.fixture
+def sample_image(tmp_path):
+    """Create a sample image file for testing."""
+    image_path = tmp_path / "test_image.png"
+    # Create a small black 1x1 pixel PNG file
+    image_content = base64.b64decode(
+        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="
+    )
+    image_path.write_bytes(image_content)
+    return image_path
+
+
+class TestImageUtils:
+    def test_convert_image_to_base64_success(self, sample_image):
+        """Test successful conversion of image to base64."""
+        base64_str = convert_image_to_base64(sample_image)
+        assert isinstance(base64_str, str)
+        # Verify it's valid base64
+        assert base64.b64decode(base64_str)
+
+    def test_convert_image_to_base64_empty_path(self):
+        """Test conversion with empty path."""
+        with pytest.raises(ValueError, match="Image path cannot be empty"):
+            convert_image_to_base64("")
+
+    def test_convert_image_to_base64_nonexistent_file(self):
+        """Test conversion with non-existent file."""
+        with pytest.raises(FileNotFoundError, match="Image file not found"):
+            convert_image_to_base64("nonexistent.png")
+
+    def test_convert_image_to_base64_directory(self, tmp_path):
+        """Test conversion with directory path instead of file."""
+        with pytest.raises(ValueError, match="Path is not a file"):
+            convert_image_to_base64(tmp_path)
+
+    def test_create_data_url_success(self, sample_image):
+        """Test successful creation of data URL."""
+        data_url = create_data_url(sample_image)
+        assert data_url.startswith("data:image/png;base64,")
+        # Verify the base64 part is valid
+        base64_part = data_url.split(",")[1]
+        assert base64.b64decode(base64_part)
+
+    def test_create_data_url_with_custom_mime(self, sample_image):
+        """Test creation of data URL with custom MIME type."""
+        custom_mime = "image/custom"
+        data_url = create_data_url(sample_image, mime_type=custom_mime)
+        assert data_url.startswith(f"data:{custom_mime};base64,")
+
+    def test_create_data_url_invalid_file(self):
+        """Test creation of data URL with invalid file."""
+        with pytest.raises(FileNotFoundError):
+            create_data_url("nonexistent.jpg")
+
+    def test_create_data_url_unrecognized_extension(self, tmp_path):
+        """Test creation of data URL with unrecognized file extension."""
+        invalid_file = tmp_path / "test.unknown"
+        invalid_file.touch()
+        with pytest.raises(ValueError, match="Could not determine MIME type"):
+            create_data_url(invalid_file)