jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Update SplitTextComponent in langflow/helpers***

Browse source
This commit is contained in:
Rodrigo Nader 2024-03-30 19:06:23 -03:00
commit 7eda0fff74
1 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

11
src/backend/base/langflow/components/helpers/SplitText.py
View file

@ -9,7 +9,7 @@ from langchain_core.documents import Document
from langflow.interface.custom.custom_component import CustomComponent from langflow.interface.custom.custom_component import CustomComponent
from langflow.schema import Record from langflow.schema import Record
from langflow.field_typing import Text from langflow.field_typing import Text
from langflow.utils.util import unescape_string from langflow.utils.util import build_loader_repr_from_records, unescape_string
class SplitTextComponent(CustomComponent): class SplitTextComponent(CustomComponent):
@ -25,18 +25,18 @@ class SplitTextComponent(CustomComponent):
}, },
"separators": { "separators": {
"display_name": "Separators", "display_name": "Separators",
"info": 'The characters to split on.\nIf left empty defaults to [" "].', "info": 'The characters to split on. Defaults to [" "].',
"is_list": True, "is_list": True,
}, },
"chunk_size": { "chunk_size": {
"display_name": "Chunk Size", "display_name": "Max Chunk Size",
"info": "The maximum length of each chunk.", "info": "The maximum length (in number of characters) of each chunk.",
"field_type": "int", "field_type": "int",
"value": 1000, "value": 1000,
}, },
"chunk_overlap": { "chunk_overlap": {
"display_name": "Chunk Overlap", "display_name": "Chunk Overlap",
"info": "The amount of overlap between chunks.", "info": "The amount of character overlap between chunks.",
"field_type": "int", "field_type": "int",
"value": 200, "value": 200,
}, },
@ -54,6 +54,7 @@ class SplitTextComponent(CustomComponent):
chunk_overlap: Optional[int] = 200, chunk_overlap: Optional[int] = 200,
recursive: bool = False, recursive: bool = False,
) -> list[Record]: ) -> list[Record]:
separators = [unescape_string(x) for x in separators] separators = [unescape_string(x) for x in separators]
# Make sure chunk_size and chunk_overlap are ints # Make sure chunk_size and chunk_overlap are ints