From 36212884e4f3f8f7e11b9b47e6dbc572783f2a9c Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Wed, 28 Jun 2023 18:16:13 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(loading.py):=20refactor=20in?= =?UTF-8?q?stantiate=5Ftextsplitter=20function=20to=20improve=20readabilit?= =?UTF-8?q?y=20and=20remove=20unnecessary=20code=20The=20`instantiate=5Fte?= =?UTF-8?q?xtsplitter`=20function=20has=20been=20refactored=20to=20improve?= =?UTF-8?q?=20readability=20and=20remove=20unnecessary=20code.=20The=20con?= =?UTF-8?q?dition=20for=20creating=20the=20`text=5Fsplitter`=20object=20ha?= =?UTF-8?q?s=20been=20simplified=20and=20the=20`separator=5Ftype`=20parame?= =?UTF-8?q?ter=20is=20now=20removed=20from=20the=20`params`=20dictionary?= =?UTF-8?q?=20if=20it=20exists.=20Additionally,=20the=20`language`=20param?= =?UTF-8?q?eter=20is=20now=20passed=20as=20an=20instance=20of=20the=20`Lan?= =?UTF-8?q?guage`=20class=20from=20the=20`langchain.text=5Fsplitter`=20mod?= =?UTF-8?q?ule.=20This=20change=20ensures=20that=20the=20`text=5Fsplitter`?= =?UTF-8?q?=20object=20is=20created=20correctly=20and=20the=20`split=5Fdoc?= =?UTF-8?q?uments`=20method=20is=20called=20with=20the=20appropriate=20par?= =?UTF-8?q?ameters.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/initialize/loading.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/backend/langflow/interface/initialize/loading.py b/src/backend/langflow/interface/initialize/loading.py index 14642828a..3ddb64c24 100644 --- a/src/backend/langflow/interface/initialize/loading.py +++ b/src/backend/langflow/interface/initialize/loading.py @@ -221,14 +221,17 @@ def instantiate_textsplitter( ) from exc if ( - "separator_type" in params - and params["separator_type"] == "Text" - or "separator_type" not in params - ): + "separator_type" in params and params["separator_type"] == "Text" + ) or "separator_type" not in params: + params.pop("separator_type", None) text_splitter = class_object(**params) else: - params["language"] = params.pop("separator_type", None) + from langchain.text_splitter import Language + + language = params.pop("separator_type", None) + params["language"] = Language(language) params.pop("separators", None) + text_splitter = class_object.from_language(**params) return text_splitter.split_documents(documents)