From 5798750b32776ddce156b68a28b2382b54f1fd23 Mon Sep 17 00:00:00 2001 From: Gabriel Almeida Date: Tue, 30 May 2023 23:38:57 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20feat(config.yaml,=20base.py):=20?= =?UTF-8?q?enable=20RecursiveCharacterTextSplitter=20and=20update=20TextSp?= =?UTF-8?q?litterCreator=20to=20handle=20it=20The=20RecursiveCharacterText?= =?UTF-8?q?Splitter=20is=20now=20enabled=20in=20the=20config.yaml=20file.?= =?UTF-8?q?=20The=20TextSplitterCreator=20class=20in=20base.py=20has=20bee?= =?UTF-8?q?n=20updated=20to=20handle=20the=20RecursiveCharacterTextSplitte?= =?UTF-8?q?r=20by=20changing=20the=20separator=20name=20to=20"separators"?= =?UTF-8?q?=20instead=20of=20"separator".=20This=20is=20because=20Recursiv?= =?UTF-8?q?eCharacterTextSplitter=20takes=20a=20list=20of=20separators=20i?= =?UTF-8?q?nstead=20of=20a=20single=20separator.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/config.yaml | 2 +- src/backend/langflow/interface/text_splitters/base.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/backend/langflow/config.yaml b/src/backend/langflow/config.yaml index 4060d1f3e..5e6a5cbed 100644 --- a/src/backend/langflow/config.yaml +++ b/src/backend/langflow/config.yaml @@ -65,7 +65,7 @@ prompts: - ZeroShotPrompt textsplitters: - CharacterTextSplitter - # - RecursiveCharacterTextSplitter + - RecursiveCharacterTextSplitter # - LatexTextSplitter # - PythonCodeTextSplitter toolkits: diff --git a/src/backend/langflow/interface/text_splitters/base.py b/src/backend/langflow/interface/text_splitters/base.py index fbacae4f9..58468a593 100644 --- a/src/backend/langflow/interface/text_splitters/base.py +++ b/src/backend/langflow/interface/text_splitters/base.py @@ -25,14 +25,18 @@ class TextSplitterCreator(LangChainTypeCreator): "show": True, "name": "documents", } + if name == "RecursiveCharacterTextSplitter": + separator_name = "separators" + else: + separator_name = "separator" - signature["template"]["separator"] = { + signature["template"][separator_name] = { "type": "str", "required": True, "show": True, "value": ".", - "name": "separator", - "display_name": "Separator", + "name": separator_name, + "display_name": separator_name.title(), } signature["template"]["chunk_size"] = {