🐛 fix(loading.py): fix issue with separators being escaped strings by decoding them using unicode-escape encoding
This commit is contained in:
parent
748b4772ed
commit
8f9bfd2a7c
1 changed files with 6 additions and 0 deletions
|
|
@ -361,6 +361,12 @@ def instantiate_textsplitter(
|
|||
"separator_type" in params and params["separator_type"] == "Text"
|
||||
) or "separator_type" not in params:
|
||||
params.pop("separator_type", None)
|
||||
# separators might come in as an escaped string like \\n
|
||||
# so we need to convert it to a string
|
||||
if "separators" in params:
|
||||
params["separators"] = (
|
||||
params["separators"].encode().decode("unicode-escape")
|
||||
)
|
||||
text_splitter = class_object(**params)
|
||||
else:
|
||||
from langchain.text_splitter import Language
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue