refactor: Update URLComponent to use 'text' key in Data objects
This commit is contained in:
parent
503437c73a
commit
791c55c2db
2 changed files with 4 additions and 4 deletions
|
|
@ -62,6 +62,6 @@ class URLComponent(Component):
|
|||
urls = [self.ensure_url(url.strip()) for url in self.urls if url.strip()]
|
||||
loader = WebBaseLoader(web_paths=urls, encoding="utf-8")
|
||||
docs = loader.load()
|
||||
data = [Data(text_key="content", content=doc.page_content, **doc.metadata) for doc in docs]
|
||||
data = [Data(text_key="text", content=doc.page_content, **doc.metadata) for doc in docs]
|
||||
self.status = data
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -73,15 +73,15 @@ class SplitTextComponent(Component):
|
|||
for chunk in chunks:
|
||||
buffer += chunk
|
||||
while len(buffer) >= max_chunk_size:
|
||||
results.append(Data(data={"parent": text, "chunk": buffer[:max_chunk_size]}))
|
||||
results.append(Data(data={"parent": text, "text": buffer[:max_chunk_size]}))
|
||||
buffer = buffer[max_chunk_size:]
|
||||
if len(buffer) >= min_chunk_size:
|
||||
results.append(Data(data={"parent": text, "chunk": buffer}))
|
||||
results.append(Data(data={"parent": text, "text": buffer}))
|
||||
buffer = ""
|
||||
|
||||
# Handle any remaining text that may not meet the min_chunk_size requirement
|
||||
if buffer:
|
||||
results.append(Data(data={"parent": text, "chunk": buffer}))
|
||||
results.append(Data(data={"parent": text, "text": buffer}))
|
||||
|
||||
self.status = results
|
||||
return results
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue