🐛 fix(loading.py): fix issue with separators being escaped strings by decoding them using unicode-escape encoding

This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-07-10 13:50:45 -03:00
commit 8f9bfd2a7c

View file

@ -361,6 +361,12 @@ def instantiate_textsplitter(
"separator_type" in params and params["separator_type"] == "Text"
) or "separator_type" not in params:
params.pop("separator_type", None)
# separators might come in as an escaped string like \\n
# so we need to convert it to a string
if "separators" in params:
params["separators"] = (
params["separators"].encode().decode("unicode-escape")
)
text_splitter = class_object(**params)
else:
from langchain.text_splitter import Language