From 2d77aaaa15f4055d0e0325309a98e7ccc4c08abd Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Mon, 26 Jun 2023 07:58:49 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(loading.py):=20fix=20instant?= =?UTF-8?q?iation=20of=20TextSplitter=20by=20removing=20unnecessary=20chec?= =?UTF-8?q?k=20for=20RecursiveCharacterTextSplitter=20The=20import=20of=20?= =?UTF-8?q?RecursiveCharacterTextSplitter=20was=20removed=20as=20it=20was?= =?UTF-8?q?=20not=20being=20used=20in=20the=20code.=20The=20instantiation?= =?UTF-8?q?=20of=20TextSplitter=20was=20fixed=20by=20removing=20the=20unne?= =?UTF-8?q?cessary=20check=20for=20RecursiveCharacterTextSplitter=20and=20?= =?UTF-8?q?simplifying=20the=20code.=20=F0=9F=94=A5=20refactor(loading.py)?= =?UTF-8?q?:=20remove=20unused=20import=20of=20RecursiveCharacterTextSplit?= =?UTF-8?q?ter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../langflow/interface/initialize/loading.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/backend/langflow/interface/initialize/loading.py b/src/backend/langflow/interface/initialize/loading.py index d362ab844..756588058 100644 --- a/src/backend/langflow/interface/initialize/loading.py +++ b/src/backend/langflow/interface/initialize/loading.py @@ -16,7 +16,6 @@ from langflow.interface.toolkits.base import toolkits_creator from langflow.interface.chains.base import chain_creator from langflow.interface.utils import load_file_into_dict from langflow.utils import validate -from langchain.text_splitter import TextSplitter, RecursiveCharacterTextSplitter from langchain.chains.base import Chain from langchain.vectorstores.base import VectorStore from langchain.document_loaders.base import BaseLoader @@ -193,7 +192,7 @@ def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict): def instantiate_textsplitter( - class_object: Type[TextSplitter], + class_object, params: Dict, ): try: @@ -204,13 +203,12 @@ def instantiate_textsplitter( "Try changing the chunk_size of the Text Splitter." ) from exc - if class_object is RecursiveCharacterTextSplitter: - if "separator_type" in params and params["separator_type"] == "Text": - text_splitter = class_object(**params) - else: - params["language"] = params.pop("separator_type", None) - params.pop("separators", None) - text_splitter = class_object.from_language(**params) + if "separator_type" in params and params["separator_type"] == "Text": + text_splitter = class_object(**params) + else: + params["language"] = params.pop("separator_type", None) + params.pop("separators", None) + text_splitter = class_object.from_language(**params) return text_splitter.split_documents(documents)