jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

🚀 feat(textsplitters.py): add a field for separator type in RecursiveCharacterTextSplitter

Browse source 
The RecursiveCharacterTextSplitter class in textsplitters.py now has a new field called separator_type. This field is used to specify the type of separator to be used in the splitter. The separator_type field is a string and can take any value from the Language enum or "Text". This change was made to improve the flexibility of the RecursiveCharacterTextSplitter class.
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-06-25 18:57:45 -03:00
commit 005e7ec51c
1 changed files with 17 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/backend/langflow/template/frontend_node/textsplitters.py
View file

@ -1,5 +1,6 @@
from langflow.template.field.base import TemplateField
from langflow.template.frontend_node.base import FrontendNode
from langchain.text_splitter import Language
class TextSplittersFrontendNode(FrontendNode):
@ -17,6 +18,22 @@ class TextSplittersFrontendNode(FrontendNode):
name = "separator"
elif self.template.type_name == "RecursiveCharacterTextSplitter":
name = "separators"
# Add a field for type of separator
# which will have Text or any value from the
# Language enum
self.template.add_field(
TemplateField(
field_type="str",
required=True,
show=True,
name="separator_type",
advanced=False,
is_list=True,
options=[x.value for x in Language],
value="Text",
display_name="Separator Type",
)
)
self.template.add_field(
TemplateField(
field_type="str",