From 002430100c9204a66aa5d436aaa61fd4f84aa743 Mon Sep 17 00:00:00 2001
From: Rodrigo <rodrigosilvanader@gmail.com>
Date: Sun, 16 Jun 2024 16:46:14 -0300
Subject: [PATCH] Refactor components and update JSON output for clarity

---
 .../base/langflow/components/data/URL.py      |  2 +-
 ...ataFromRecord.py => ExtractKeyFromData.py} |  2 +-
 .../components/experimental/SplitText.py      | 35 ++++++++++++++++---
 .../components/experimental/__init__.py       |  2 +-
 4 files changed, 33 insertions(+), 8 deletions(-)
 rename src/backend/base/langflow/components/experimental/{ExtractDataFromRecord.py => ExtractKeyFromData.py} (96%)

diff --git a/src/backend/base/langflow/components/data/URL.py b/src/backend/base/langflow/components/data/URL.py
index d27d17fe6..5155ac4aa 100644
--- a/src/backend/base/langflow/components/data/URL.py
+++ b/src/backend/base/langflow/components/data/URL.py
@@ -60,7 +60,7 @@ class URLComponent(Component):
 
     def fetch_content(self) -> Data:
         urls = [self.ensure_url(url.strip()) for url in self.urls if url.strip()]
-        loader = WebBaseLoader(web_paths=urls)
+        loader = WebBaseLoader(web_paths=urls, encoding="utf-8")
         docs = loader.load()
         data = [Data(content=doc.page_content, **doc.metadata) for doc in docs]
         self.status = data
diff --git a/src/backend/base/langflow/components/experimental/ExtractDataFromRecord.py b/src/backend/base/langflow/components/experimental/ExtractKeyFromData.py
similarity index 96%
rename from src/backend/base/langflow/components/experimental/ExtractDataFromRecord.py
rename to src/backend/base/langflow/components/experimental/ExtractKeyFromData.py
index f07e8de29..a8ca78262 100644
--- a/src/backend/base/langflow/components/experimental/ExtractDataFromRecord.py
+++ b/src/backend/base/langflow/components/experimental/ExtractKeyFromData.py
@@ -2,7 +2,7 @@ from langflow.custom import CustomComponent
 from langflow.schema import Data
 
 
-class ExtractKeyFromdataComponent(CustomComponent):
+class ExtractKeyFromDataComponent(CustomComponent):
     display_name = "Extract Key From Data"
     description = "Extracts a key from a data."
     beta: bool = True
diff --git a/src/backend/base/langflow/components/experimental/SplitText.py b/src/backend/base/langflow/components/experimental/SplitText.py
index 1f294bfd7..70c3969e5 100644
--- a/src/backend/base/langflow/components/experimental/SplitText.py
+++ b/src/backend/base/langflow/components/experimental/SplitText.py
@@ -9,7 +9,7 @@ from langflow.utils.util import unescape_string
 
 class SplitContentComponent(Component):
     display_name: str = "Split Content"
-    description: str = "Split textual content into chunks of a specified length."
+    description: str = "Split textual content into chunks based on specified criteria."
     icon = "split"
 
     inputs = [
@@ -35,7 +35,21 @@ class SplitContentComponent(Component):
         IntInput(
             name="chunk_size",
             display_name="Chunk Size",
-            info="The maximum length (in number of characters) of each chunk. Defaults to 0 (no chunking).",
+            info="The target length (in number of characters) of each chunk.",
+            value=0,
+            advanced=True
+        ),
+        IntInput(
+            name="min_chunk_size",
+            display_name="Minimum Chunk Size",
+            info="The minimum size of chunks. Smaller chunks will be merged.",
+            value=0,
+            advanced=True
+        ),
+        IntInput(
+            name="max_chunk_size",
+            display_name="Maximum Chunk Size",
+            info="The maximum size of chunks. Larger chunks will be split.",
             value=0,
             advanced=True
         ),
@@ -46,12 +60,14 @@ class SplitContentComponent(Component):
     ]
 
     def split_text(self) -> List[Data]:
-        # TODO: Add minimum chunk size - can be removed or merged
         data = self.data if isinstance(self.data, list) else [self.data]
         content_key = self.content_key
         separator = unescape_string(self.separator)
         chunk_size = self.chunk_size
+        min_chunk_size = self.min_chunk_size
+        max_chunk_size = self.max_chunk_size
         results = []
+        buffer = ""
 
         for row in data:
             content = row.data.get(content_key, '')
@@ -61,8 +77,17 @@ class SplitContentComponent(Component):
                 chunks = content.split(separator)
 
             for chunk in chunks:
-                if chunk.strip():
-                    results.append(Data(data={"parent": content, "text": chunk}))
+                buffer += chunk
+                while len(buffer) >= max_chunk_size:
+                    results.append(Data(data={"parent": content, "chunk": buffer[:max_chunk_size]}))
+                    buffer = buffer[max_chunk_size:]
+                if len(buffer) >= min_chunk_size:
+                    results.append(Data(data={"parent": content, "chunk": buffer}))
+                    buffer = ""
+
+        # Handle any remaining content that may not meet the min_chunk_size requirement
+        if buffer:
+            results.append(Data(data={"parent": content, "chunk": buffer}))
 
         self.status = results
         return results
diff --git a/src/backend/base/langflow/components/experimental/__init__.py b/src/backend/base/langflow/components/experimental/__init__.py
index d67a125c8..1e7576a90 100644
--- a/src/backend/base/langflow/components/experimental/__init__.py
+++ b/src/backend/base/langflow/components/experimental/__init__.py
@@ -1,6 +1,6 @@
 from .AgentComponent import AgentComponent
 from .ClearMessageHistory import ClearMessageHistoryComponent
-from .ExtractDataFromData import ExtractKeyFromDataComponent
+from .ExtractKeyFromData import ExtractKeyFromDataComponent
 from .FlowTool import FlowToolComponent
 from .Listen import ListenComponent
 from .ListFlows import ListFlowsComponent