🐛 fix(RecursiveCharacterTextSplitter.py): uncomment line to assign repr_value to separators variable

 feat(util.py): add build_loader_repr_from_documents function to generate a representation string from a list of documents
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-09-14 21:37:37 -03:00
commit 44466dc2d1
2 changed files with 13 additions and 2 deletions

View file

@ -1,6 +1,7 @@
from typing import Optional
from langflow import CustomComponent
from langchain.schema import Document
from langflow.utils.util import build_loader_repr_from_documents
class RecursiveCharacterTextSplitterComponent(CustomComponent):
@ -74,5 +75,5 @@ class RecursiveCharacterTextSplitterComponent(CustomComponent):
)
docs = splitter.split_documents(documents)
self.repr_value = separators
self.repr_value = build_loader_repr_from_documents(docs)
return docs

View file

@ -2,12 +2,13 @@ import re
import inspect
import importlib
from functools import wraps
from typing import Optional, Dict, Any, Union
from typing import List, Optional, Dict, Any, Union
from docstring_parser import parse
from langflow.template.frontend_node.constants import FORCE_SHOW_FIELDS
from langflow.utils import constants
from langchain.schema import Document
def build_template_from_function(
@ -456,3 +457,12 @@ def add_options_to_field(
value["options"] = options_map[class_name]
value["list"] = True
value["value"] = options_map[class_name][0]
def build_loader_repr_from_documents(documents: List[Document]) -> str:
if documents:
avg_length = sum(len(doc.page_content) for doc in documents) / len(documents)
return f"""{len(documents)} documents
\nAvg. Document Length (characters): {int(avg_length)}
Documents: {documents[:3]}..."""
return "0 documents"