refactor: Update URLComponent to include text_key in Data objects

This commit is contained in:
ogabrielluiz 2024-06-17 19:35:26 -03:00
commit 65e888b7a8

View file

@ -62,6 +62,6 @@ class URLComponent(Component):
urls = [self.ensure_url(url.strip()) for url in self.urls if url.strip()]
loader = WebBaseLoader(web_paths=urls, encoding="utf-8")
docs = loader.load()
data = [Data(content=doc.page_content, **doc.metadata) for doc in docs]
data = [Data(text_key="content", content=doc.page_content, **doc.metadata) for doc in docs]
self.status = data
return data