feat: add multi-file support and rename NVIDIA Ingest to NeMo Retriever Extraction (#7495)
* nv-ingest multi-file support * [autofix.ci] apply automated fixes * fix linter error --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
parent
eb814b29c0
commit
dff2606fa0
1 changed files with 30 additions and 44 deletions
|
|
@ -1,44 +1,34 @@
|
||||||
from pathlib import Path
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from langflow.custom import Component
|
from langflow.base.data import BaseFileComponent
|
||||||
from langflow.io import BoolInput, DropdownInput, FileInput, IntInput, MessageTextInput, Output
|
from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput
|
||||||
from langflow.schema import Data
|
from langflow.schema import Data
|
||||||
|
|
||||||
|
|
||||||
class NvidiaIngestComponent(Component):
|
class NvidiaIngestComponent(BaseFileComponent):
|
||||||
display_name = "NVIDIA Ingest"
|
display_name = "NVIDIA Retriever Extraction"
|
||||||
description = "Process, transform, and store data."
|
description = "Multi-modal data extraction from documents using NVIDIA's NeMo API."
|
||||||
documentation: str = "https://github.com/NVIDIA/nv-ingest/tree/main/docs"
|
documentation: str = "https://docs.nvidia.com/nemo/retriever/extraction/overview/"
|
||||||
icon = "NVIDIA"
|
icon = "NVIDIA"
|
||||||
name = "NVIDIAIngest"
|
|
||||||
beta = True
|
beta = True
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE
|
from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE
|
||||||
|
|
||||||
file_types = list(EXTENSION_TO_DOCUMENT_TYPE.keys())
|
VALID_EXTENSIONS = list(EXTENSION_TO_DOCUMENT_TYPE.keys())
|
||||||
supported_file_types_info = f"Supported file types: {', '.join(file_types)}"
|
|
||||||
except ImportError:
|
except ImportError:
|
||||||
msg = (
|
msg = (
|
||||||
"NVIDIA Ingest dependencies missing. "
|
"NVIDIA Retriever Extraction (nv-ingest) dependencies missing. "
|
||||||
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
|
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
|
||||||
)
|
)
|
||||||
file_types = []
|
VALID_EXTENSIONS = [msg]
|
||||||
supported_file_types_info = msg
|
|
||||||
|
|
||||||
inputs = [
|
inputs = [
|
||||||
|
*BaseFileComponent._base_inputs,
|
||||||
MessageTextInput(
|
MessageTextInput(
|
||||||
name="base_url",
|
name="base_url",
|
||||||
display_name="NVIDIA Ingestion URL",
|
display_name="Base URL",
|
||||||
info="The URL of the NVIDIA Ingestion API.",
|
info="The URL of the NVIDIA NeMo Retriever Extraction API.",
|
||||||
),
|
|
||||||
FileInput(
|
|
||||||
name="path",
|
|
||||||
display_name="Path",
|
|
||||||
file_types=file_types,
|
|
||||||
info=supported_file_types_info,
|
|
||||||
required=True,
|
|
||||||
),
|
),
|
||||||
BoolInput(
|
BoolInput(
|
||||||
name="extract_text",
|
name="extract_text",
|
||||||
|
|
@ -114,53 +104,47 @@ class NvidiaIngestComponent(Component):
|
||||||
]
|
]
|
||||||
|
|
||||||
outputs = [
|
outputs = [
|
||||||
Output(display_name="Data", name="data", method="load_file"),
|
*BaseFileComponent._base_outputs,
|
||||||
]
|
]
|
||||||
|
|
||||||
def load_file(self) -> list[Data]:
|
def process_files(self, file_list: list[BaseFileComponent.BaseFile]) -> list[BaseFileComponent.BaseFile]:
|
||||||
try:
|
try:
|
||||||
from nv_ingest_client.client import Ingestor
|
from nv_ingest_client.client import Ingestor
|
||||||
except ImportError as e:
|
except ImportError as e:
|
||||||
msg = (
|
msg = (
|
||||||
"NVIDIA Ingest dependencies missing. "
|
"NVIDIA Retriever Extraction (nv-ingest) dependencies missing. "
|
||||||
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
|
"Please install them using your package manager. (e.g. uv pip install langflow[nv-ingest])"
|
||||||
)
|
)
|
||||||
raise ImportError(msg) from e
|
raise ImportError(msg) from e
|
||||||
|
|
||||||
self.base_url: str | None = self.base_url.strip() if self.base_url else None
|
self.base_url: str | None = self.base_url.strip() if self.base_url else None
|
||||||
|
|
||||||
if not self.path:
|
if not file_list:
|
||||||
err_msg = "Upload a file to use this component."
|
err_msg = "No files to process."
|
||||||
self.log(err_msg, name="NVIDIAIngestComponent")
|
self.log(err_msg)
|
||||||
raise ValueError(err_msg)
|
raise ValueError(err_msg)
|
||||||
|
|
||||||
resolved_path = self.resolve_path(self.path)
|
file_paths = [str(file.path) for file in file_list]
|
||||||
extension = Path(resolved_path).suffix[1:].lower()
|
|
||||||
if extension not in self.file_types:
|
|
||||||
err_msg = f"Unsupported file type: {extension}"
|
|
||||||
self.log(err_msg, name="NVIDIAIngestComponent")
|
|
||||||
raise ValueError(err_msg)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
parsed_url = urlparse(self.base_url)
|
parsed_url = urlparse(self.base_url)
|
||||||
if not parsed_url.hostname or not parsed_url.port:
|
if not parsed_url.hostname or not parsed_url.port:
|
||||||
err_msg = "Invalid URL: Missing hostname or port."
|
err_msg = "Invalid URL: Missing hostname or port."
|
||||||
self.log(err_msg, name="NVIDIAIngestComponent")
|
self.log(err_msg)
|
||||||
raise ValueError(err_msg)
|
raise ValueError(err_msg)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(f"Error parsing URL: {e}", name="NVIDIAIngestComponent")
|
self.log(f"Error parsing URL: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
self.log(
|
self.log(
|
||||||
f"Creating Ingestor for host: {parsed_url.hostname!r}, port: {parsed_url.port!r}",
|
f"Creating Ingestor for host: {parsed_url.hostname!r}, port: {parsed_url.port!r}",
|
||||||
name="NVIDIAIngestComponent",
|
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
from nv_ingest_client.client import Ingestor
|
from nv_ingest_client.client import Ingestor
|
||||||
|
|
||||||
ingestor = (
|
ingestor = (
|
||||||
Ingestor(message_client_hostname=parsed_url.hostname, message_client_port=parsed_url.port)
|
Ingestor(message_client_hostname=parsed_url.hostname, message_client_port=parsed_url.port)
|
||||||
.files(resolved_path)
|
.files(file_paths)
|
||||||
.extract(
|
.extract(
|
||||||
extract_text=self.extract_text,
|
extract_text=self.extract_text,
|
||||||
extract_tables=self.extract_tables,
|
extract_tables=self.extract_tables,
|
||||||
|
|
@ -170,7 +154,7 @@ class NvidiaIngestComponent(Component):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(f"Error creating Ingestor: {e}", name="NVIDIAIngestComponent")
|
self.log(f"Error creating Ingestor: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
if self.split_text:
|
if self.split_text:
|
||||||
|
|
@ -185,12 +169,12 @@ class NvidiaIngestComponent(Component):
|
||||||
try:
|
try:
|
||||||
result = ingestor.ingest()
|
result = ingestor.ingest()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
self.log(f"Error during ingestion: {e}", name="NVIDIAIngestComponent")
|
self.log(f"Error during ingestion: {e}")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
self.log(f"Results: {result}", name="NVIDIAIngestComponent")
|
self.log(f"Results: {result}")
|
||||||
|
|
||||||
data = []
|
data: list[Data | None] = []
|
||||||
document_type_text = "text"
|
document_type_text = "text"
|
||||||
document_type_structured = "structured"
|
document_type_structured = "structured"
|
||||||
|
|
||||||
|
|
@ -226,7 +210,9 @@ class NvidiaIngestComponent(Component):
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# image is not yet supported; skip if encountered
|
# image is not yet supported; skip if encountered
|
||||||
self.log(f"Unsupported document type: {document_type}", name="NVIDIAIngestComponent")
|
self.log(f"Unsupported document type: {document_type}")
|
||||||
|
|
||||||
self.status = data or "No data"
|
self.status = data or "No data"
|
||||||
return data
|
|
||||||
|
# merge processed data with BaseFile objects
|
||||||
|
return self.rollup_data(file_list, data)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue