🐛 fix(RecursiveCharacterTextSplitter.py): uncomment line to assign repr_value to separators variable

✨ feat(util.py): add build_loader_repr_from_documents function to generate a representation string from a list of documents
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-09-14 21:37:37 -03:00
commit 44466dc2d1
2 changed files with 13 additions and 2 deletions

View file

@ -1,6 +1,7 @@
from typing import Optional from typing import Optional
from langflow import CustomComponent from langflow import CustomComponent
from langchain.schema import Document from langchain.schema import Document
from langflow.utils.util import build_loader_repr_from_documents
class RecursiveCharacterTextSplitterComponent(CustomComponent): class RecursiveCharacterTextSplitterComponent(CustomComponent):
@ -74,5 +75,5 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent):
) )
docs = splitter.split_documents(documents) docs = splitter.split_documents(documents)
self.repr_value = separators self.repr_value = build_loader_repr_from_documents(docs)
return docs return docs

View file

@ -2,12 +2,13 @@ import re
import inspect import inspect
import importlib import importlib
from functools import wraps from functools import wraps
from typing import Optional, Dict, Any, Union from typing import List, Optional, Dict, Any, Union
from docstring_parser import parse from docstring_parser import parse
from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
from langflow.utils import constants from langflow.utils import constants
from langchain.schema import Document
def build_template_from_function( def build_template_from_function(
@ -456,3 +457,12 @@ def add_options_to_field(
value["options"] = options_map[class_name] value["options"] = options_map[class_name]
value["list"] = True value["list"] = True
value["value"] = options_map[class_name][0] value["value"] = options_map[class_name][0]
def build_loader_repr_from_documents(documents: List[Document]) -> str:
if documents:
avg_length = sum(len(doc.page_content) for doc in documents) / len(documents)
return f"""{len(documents)} documents
\nAvg. Document Length (characters): {int(avg_length)}
Documents: {documents[:3]}..."""
return "0 documents"