Refactor document loaders to support metadata

This commit is contained in:
anovazzi1 2024-01-15 18:34:22 -03:00
commit 59a1370c2d
2 changed files with 18 additions and 10 deletions

View file

@ -16,7 +16,11 @@ class AZLyricsLoaderComponent(CustomComponent):
} }
def build(self, metadata: Optional[Dict] = None, web_path: str = "") -> Document: def build(self, metadata: Optional[Dict] = None, web_path: str = "") -> Document:
# Assuming there is a class AZLyricsLoader that takes metadata and web_path as parameters documents = AZLyricsLoader(web_path=web_path).load()
# and returns a Document object. Replace AZLyricsLoader with the actual class name if different. if(metadata):
# The import statement for AZLyricsLoader is assumed to be added above. for document in documents:
return AZLyricsLoader(metadata=metadata, web_path=web_path) if not document.metadata:
document.metadata = metadata
else:
document.metadata.update(metadata)
return documents

View file

@ -1,6 +1,7 @@
from langflow import CustomComponent from langflow import CustomComponent
from langflow.field_typing import Document from langflow.field_typing import Document
from typing import Optional, Dict from typing import Optional, Dict
from langchain_community.document_loaders.airbyte_json import AirbyteJSONLoader
class AirbyteJSONLoaderComponent(CustomComponent): class AirbyteJSONLoaderComponent(CustomComponent):
@ -14,8 +15,7 @@ class AirbyteJSONLoaderComponent(CustomComponent):
return { return {
"file_path": { "file_path": {
"display_name": "File Path", "display_name": "File Path",
"type": "file", "file_types": [".json"],
"fileTypes": ["json"],
"required": True, "required": True,
"field_type": "file", "field_type": "file",
}, },
@ -27,7 +27,11 @@ class AirbyteJSONLoaderComponent(CustomComponent):
} }
def build(self, file_path: str, metadata: Optional[Dict] = None) -> Document: def build(self, file_path: str, metadata: Optional[Dict] = None) -> Document:
# Assuming there is a function or class named AirbyteJSONLoader that takes file_path and metadata as parameters documents = AirbyteJSONLoader(file_path=file_path).load()
# and returns a Document object. Replace AirbyteJSONLoader with the actual class or function name. if(metadata):
# The actual implementation here is a placeholder and should be adapted to the real AirbyteJSONLoader class or function. for document in documents:
return AirbyteJSONLoader(file_path=file_path, metadata=metadata) if not document.metadata:
document.metadata = metadata
else:
document.metadata.update(metadata)
return documents