refactor: Update URLComponent to use 'text' key in Data objects

This commit is contained in:
Rodrigo 2024-06-17 20:13:47 -03:00
commit 791c55c2db
2 changed files with 4 additions and 4 deletions

View file

@ -62,6 +62,6 @@ class URLComponent(Component):
urls = [self.ensure_url(url.strip()) for url in self.urls if url.strip()]
loader = WebBaseLoader(web_paths=urls, encoding="utf-8")
docs = loader.load()
data = [Data(text_key="content", content=doc.page_content, **doc.metadata) for doc in docs]
data = [Data(text_key="text", content=doc.page_content, **doc.metadata) for doc in docs]
self.status = data
return data

View file

@ -73,15 +73,15 @@ class SplitTextComponent(Component):
for chunk in chunks:
buffer += chunk
while len(buffer) >= max_chunk_size:
results.append(Data(data={"parent": text, "chunk": buffer[:max_chunk_size]}))
results.append(Data(data={"parent": text, "text": buffer[:max_chunk_size]}))
buffer = buffer[max_chunk_size:]
if len(buffer) >= min_chunk_size:
results.append(Data(data={"parent": text, "chunk": buffer}))
results.append(Data(data={"parent": text, "text": buffer}))
buffer = ""
# Handle any remaining text that may not meet the min_chunk_size requirement
if buffer:
results.append(Data(data={"parent": text, "chunk": buffer}))
results.append(Data(data={"parent": text, "text": buffer}))
self.status = results
return results