jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Refactor document loader component and improve

Browse source 
error handling
This commit is contained in:
Gabriel Luiz Freitas Almeida 2023-11-28 20:23:10 -03:00
commit 3accebc0a9
1 changed files with 12 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

40
src/backend/langflow/components/documentloaders/UrlLoader.py
View file

@ -1,15 +1,8 @@
from typing import List from typing import List
from langflow import CustomComponent
from langchain.document_loaders import AZLyricsLoader
from langchain.document_loaders import CollegeConfidentialLoader
from langchain.document_loaders import GitbookLoader
from langchain.document_loaders import HNLoader
from langchain.document_loaders import IFixitLoader
from langchain.document_loaders import IMSDbLoader
from langchain.document_loaders import WebBaseLoader
from langchain import document_loaders
from langchain.schema import Document from langchain.schema import Document
from langflow import CustomComponent
class UrlLoaderComponent(CustomComponent): class UrlLoaderComponent(CustomComponent):
@ -41,22 +34,13 @@ class UrlLoaderComponent(CustomComponent):
} }
def build(self, web_path: str, loader: str) -> List[Document]: def build(self, web_path: str, loader: str) -> List[Document]:
if loader == "AZLyricsLoader": try:
loader_instance = AZLyricsLoader(web_path=web_path) # type: ignore loader_instance = getattr(document_loaders, loader)(web_path=web_path)
elif loader == "CollegeConfidentialLoader": except Exception as e:
loader_instance = CollegeConfidentialLoader(web_path=web_path) # type: ignore raise ValueError(f"No loader found for: {web_path}") from e
elif loader == "GitbookLoader": docs = loader_instance.load()
loader_instance = GitbookLoader(web_page=web_path) # type: ignore avg_length = sum(len(doc.page_content) for doc in docs if hasattr(doc, "page_content")) / len(docs)
elif loader == "HNLoader": self.status = f"""{len(docs)} documents)
loader_instance = HNLoader(web_path=web_path) # type: ignore \nAvg. Document Length (characters): {int(avg_length)}
elif loader == "IFixitLoader": Documents: {docs[:3]}..."""
loader_instance = IFixitLoader(web_path=web_path) # type: ignore return docs
elif loader == "IMSDbLoader":
loader_instance = IMSDbLoader(web_path=web_path) # type: ignore
elif loader == "WebBaseLoader":
loader_instance = WebBaseLoader(web_path=web_path) # type: ignore
if loader_instance is None:
raise ValueError(f"No loader found for: {web_path}")
return loader_instance.load()