feat: add multi-file support and rename NVIDIA Ingest to NeMo Retriever Extraction (#7495)

* nv-ingest multi-file support

* [autofix.ci] apply automated fixes

* fix linter error

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
Jeffrey Carpenter 2025-04-10 09:13:23 -07:00 • committed by GitHub
commit dff2606fa0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -1,44 +1,34 @@
from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from langflow.custom import Component from langflow.base.data import BaseFileComponent
from langflow.io import BoolInput, DropdownInput, FileInput, IntInput, MessageTextInput, Output from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput
from langflow.schema import Data from langflow.schema import Data
class NvidiaIngestComponent(Component): class NvidiaIngestComponent(BaseFileComponent):
display_name = "NVIDIA Ingest" display_name = "NVIDIA Retriever Extraction"
description = "Process, transform, and store data." description = "Multi-modal data extraction from documents using NVIDIA's NeMo API."
documentation: str = "https://github.com/NVIDIA/nv-ingest/tree/main/docs" documentation: str = "https://docs.nvidia.com/nemo/retriever/extraction/overview/"
icon = "NVIDIA" icon = "NVIDIA"
name = "NVIDIAIngest"
beta = True beta = True
try: try:
from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE
file_types = list(EXTENSION_TO_DOCUMENT_TYPE.keys()) VALID_EXTENSIONS = list(EXTENSION_TO_DOCUMENT_TYPE.keys())
supported_file_types_info = f"Supported file types: {', '.join(file_types)}"
except ImportError: except ImportError:
msg = ( msg = (
"NVIDIA Ingest dependencies missing. " "NVIDIA Retriever Extraction (nv-ingest) dependencies missing. "
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])" "Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
) )
file_types = [] VALID_EXTENSIONS = [msg]
supported_file_types_info = msg
inputs = [ inputs = [
*BaseFileComponent._base_inputs,
MessageTextInput( MessageTextInput(
name="base_url", name="base_url",
display_name="NVIDIA Ingestion URL", display_name="Base URL",
info="The URL of the NVIDIA Ingestion API.", info="The URL of the NVIDIA NeMo Retriever Extraction API.",
),
FileInput(
name="path",
display_name="Path",
file_types=file_types,
info=supported_file_types_info,
required=True,
), ),
BoolInput( BoolInput(
name="extract_text", name="extract_text",
@ -114,53 +104,47 @@ class NvidiaIngestComponent(Component):
] ]
outputs = [ outputs = [
Output(display_name="Data", name="data", method="load_file"), *BaseFileComponent._base_outputs,
] ]
def load_file(self) -> list[Data]: def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:
try: try:
from nv_ingest_client.client import Ingestor from nv_ingest_client.client import Ingestor
except ImportError as e: except ImportError as e:
msg = ( msg = (
"NVIDIA Ingest dependencies missing. " "NVIDIA Retriever Extraction (nv-ingest) dependencies missing. "
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])" "Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
) )
raise ImportError(msg) from e raise ImportError(msg) from e
self.base_url: str | None = self.base_url.strip() if self.base_url else None self.base_url: str | None = self.base_url.strip() if self.base_url else None
if not self.path: if not file_list:
err_msg = "Upload a file to use this component." err_msg = "No files to process."
self.log(err_msg, name="NVIDIAIngestComponent") self.log(err_msg)
raise ValueError(err_msg) raise ValueError(err_msg)
resolved_path = self.resolve_path(self.path) file_paths = [str(file.path) for file in file_list]
extension = Path(resolved_path).suffix[1:].lower()
if extension not in self.file_types:
err_msg = f"Unsupported file type: {extension}"
self.log(err_msg, name="NVIDIAIngestComponent")
raise ValueError(err_msg)
try: try:
parsed_url = urlparse(self.base_url) parsed_url = urlparse(self.base_url)
if not parsed_url.hostname or not parsed_url.port: if not parsed_url.hostname or not parsed_url.port:
err_msg = "Invalid URL: Missing hostname or port." err_msg = "Invalid URL: Missing hostname or port."
self.log(err_msg, name="NVIDIAIngestComponent") self.log(err_msg)
raise ValueError(err_msg) raise ValueError(err_msg)
except Exception as e: except Exception as e:
self.log(f"Error parsing URL: {e}", name="NVIDIAIngestComponent") self.log(f"Error parsing URL: {e}")
raise raise
self.log( self.log(
f"Creating Ingestor for host: {parsed_url.hostname!r}, port: {parsed_url.port!r}", f"Creating Ingestor for host: {parsed_url.hostname!r}, port: {parsed_url.port!r}",
name="NVIDIAIngestComponent",
) )
try: try:
from nv_ingest_client.client import Ingestor from nv_ingest_client.client import Ingestor
ingestor = ( ingestor = (
Ingestor(message_client_hostname=parsed_url.hostname, message_client_port=parsed_url.port) Ingestor(message_client_hostname=parsed_url.hostname, message_client_port=parsed_url.port)
.files(resolved_path) .files(file_paths)
.extract( .extract(
extract_text=self.extract_text, extract_text=self.extract_text,
extract_tables=self.extract_tables, extract_tables=self.extract_tables,
@ -170,7 +154,7 @@ class NvidiaIngestComponent(Component):
) )
) )
except Exception as e: except Exception as e:
self.log(f"Error creating Ingestor: {e}", name="NVIDIAIngestComponent") self.log(f"Error creating Ingestor: {e}")
raise raise
if self.split_text: if self.split_text:
@ -185,12 +169,12 @@ class NvidiaIngestComponent(Component):
try: try:
result = ingestor.ingest() result = ingestor.ingest()
except Exception as e: except Exception as e:
self.log(f"Error during ingestion: {e}", name="NVIDIAIngestComponent") self.log(f"Error during ingestion: {e}")
raise raise
self.log(f"Results: {result}", name="NVIDIAIngestComponent") self.log(f"Results: {result}")
data = [] data: list[Data | None] = []
document_type_text = "text" document_type_text = "text"
document_type_structured = "structured" document_type_structured = "structured"
@ -226,7 +210,9 @@ class NvidiaIngestComponent(Component):
) )
else: else:
# image is not yet supported; skip if encountered # image is not yet supported; skip if encountered
self.log(f"Unsupported document type: {document_type}", name="NVIDIAIngestComponent") self.log(f"Unsupported document type: {document_type}")
self.status = data or "No data" self.status = data or "No data"
return data
# merge processed data with BaseFile objects
return self.rollup_data(file_list, data)