jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: NV-ingest image extraction and authentication support (#7795)

Browse source 
* nv-ingest auth workaround

* nv-ingest local dependency

* workaround for RestClient

* use forked nv-ingest-client with auth/url improvements

* update to nv-ingest-client 20250422

* image extraction

* fix splitter options

* improving defaults

* review feedback fixes

* [autofix.ci] apply automated fixes

* ruff fixes

* require base url

* [autofix.ci] apply automated fixes

* add check for empty segment

* Remove html from list of supported

* Use hardcoded list of supported types

* prayer to lint gods

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Jordan Frazier <jordan.frazier@datastax.com>
Co-authored-by: Jordan Frazier <122494242+jordanrfrazier@users.noreply.github.com>
This commit is contained in:
Jeffrey Carpenter 2025-04-28 21:38:57 -07:00 • committed by GitHub
commit 4610bc3eb4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 4506 additions and 4379 deletions
Download patch file Download diff file Expand all files Collapse all files

10
pyproject.toml
View file

@ -208,12 +208,8 @@ clickhouse-connect = [
] ]
nv-ingest = [ nv-ingest = [
# nv-ingest-client 2025.2.7.dev0 does not correctly install its "nv-ingest-api==2025.4.22.dev20250422",
# dependencies, so we need to install some manually. "nv-ingest-client==2025.4.22.dev20250422",
"nv-ingest-client==2025.2.7.dev0",
"python-pptx==0.6.23",
"pymilvus[bulk_writer,model]==2.5.0",
"llama-index-embeddings-nvidia==0.1.5",
] ]
postgresql = [ postgresql = [
@ -320,4 +316,4 @@ ignore_missing_imports = true
[build-system] [build-system]
requires = ["hatchling"] requires = ["hatchling"]
build-backend = "hatchling.build" build-backend = "hatchling.build"

244
src/backend/base/langflow/components/nvidia/nvidia_ingest.py
View file

@ -1,7 +1,14 @@
from urllib.parse import urlparse from urllib.parse import urlparse
from langflow.base.data import BaseFileComponent from langflow.base.data import BaseFileComponent
from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput from langflow.io import (
BoolInput,
DropdownInput,
FloatInput,
IntInput,
MessageTextInput,
SecretStrInput,
)
from langflow.schema import Data from langflow.schema import Data
@ -15,7 +22,8 @@ class NvidiaIngestComponent(BaseFileComponent):
try: try:
from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE from nv_ingest_client.util.file_processing.extract import EXTENSION_TO_DOCUMENT_TYPE
VALID_EXTENSIONS = list(EXTENSION_TO_DOCUMENT_TYPE.keys()) # Supported file extensions from https://github.com/NVIDIA/nv-ingest/blob/main/README.md
VALID_EXTENSIONS = ["pdf", "docx", "pptx", "jpeg", "png", "svg", "tiff", "txt"]
except ImportError: except ImportError:
msg = ( msg = (
"NVIDIA Retriever Extraction (nv-ingest) dependencies missing. " "NVIDIA Retriever Extraction (nv-ingest) dependencies missing. "
@ -29,6 +37,11 @@ class NvidiaIngestComponent(BaseFileComponent):
name="base_url", name="base_url",
display_name="Base URL", display_name="Base URL",
info="The URL of the NVIDIA NeMo Retriever Extraction API.", info="The URL of the NVIDIA NeMo Retriever Extraction API.",
required=True,
),
SecretStrInput(
name="api_key",
display_name="NVIDIA API Key",
), ),
BoolInput( BoolInput(
name="extract_text", name="extract_text",
@ -46,6 +59,12 @@ class NvidiaIngestComponent(BaseFileComponent):
name="extract_tables", name="extract_tables",
display_name="Extract Tables", display_name="Extract Tables",
info="Extract text from tables", info="Extract text from tables",
value=False,
),
BoolInput(
name="extract_images",
display_name="Extract Images",
info="Extract images from document",
value=True, value=True,
), ),
DropdownInput( DropdownInput(
@ -56,7 +75,7 @@ class NvidiaIngestComponent(BaseFileComponent):
"Support for 'block', 'line', 'span' varies by document type." "Support for 'block', 'line', 'span' varies by document type."
), ),
options=["document", "page", "block", "line", "span"], options=["document", "page", "block", "line", "span"],
value="document", # Default value value="page", # Default value
advanced=True, advanced=True,
), ),
BoolInput( BoolInput(
@ -64,43 +83,64 @@ class NvidiaIngestComponent(BaseFileComponent):
display_name="Split Text", display_name="Split Text",
info="Split text into smaller chunks", info="Split text into smaller chunks",
value=True, value=True,
),
DropdownInput(
name="split_by",
display_name="Split By",
info="How to split into chunks ('size' splits by number of characters)",
options=["page", "sentence", "word", "size"],
value="word", # Default value
advanced=True, advanced=True,
), ),
IntInput( IntInput(
name="split_length", name="chunk_size",
display_name="Split Length", display_name="Chunk size",
info="The size of each chunk based on the 'split_by' method", info="The number of tokens per chunk",
value=200, value=500,
advanced=True, advanced=True,
), ),
IntInput( IntInput(
name="split_overlap", name="chunk_overlap",
display_name="Split Overlap", display_name="Chunk Overlap",
info="Number of segments (as determined by the 'split_by' method) to overlap from previous chunk", info="Number of tokens to overlap from previous chunk",
value=20, value=150,
advanced=True, advanced=True,
), ),
IntInput( BoolInput(
name="max_character_length", name="filter_images",
display_name="Max Character Length", display_name="Filter Images",
info="Maximum number of characters in each chunk", info="Filter images (see advanced options for filtering criteria).",
value=1000,
advanced=True, advanced=True,
value=True,
), ),
IntInput( IntInput(
name="sentence_window_size", name="min_image_size",
display_name="Sentence Window Size", display_name="Minimum Image Size Filter",
info="Number of sentences to include from previous and following chunk (when split_by='sentence')", info="Minimum image width/length in pixels",
value=0, value=128,
advanced=True, advanced=True,
), ),
FloatInput(
name="min_aspect_ratio",
display_name="Minimum Aspect Ratio Filter",
info="Minimum allowed aspect ratio (width / height). Images narrower than this will be filtered out.",
value=0.2,
advanced=True,
),
FloatInput(
name="max_aspect_ratio",
display_name="Maximum Aspect Ratio Filter",
info="Maximum allowed aspect ratio (width / height). Images taller than this will be filtered out.",
value=5.0,
advanced=True,
),
BoolInput(
name="dedup_images",
display_name="Deduplicate Images",
info="Filter duplicated images.",
advanced=True,
value=True,
),
BoolInput(
name="caption_images",
display_name="Caption Images",
info="Generate captions for images using the NVIDIA captioning model.",
advanced=True,
value=True,
),
] ]
outputs = [ outputs = [
@ -117,8 +157,6 @@ class NvidiaIngestComponent(BaseFileComponent):
) )
raise ImportError(msg) from e raise ImportError(msg) from e
self.base_url: str | None = self.base_url.strip() if self.base_url else None
if not file_list: if not file_list:
err_msg = "No files to process." err_msg = "No files to process."
self.log(err_msg) self.log(err_msg)
@ -126,50 +164,70 @@ class NvidiaIngestComponent(BaseFileComponent):
file_paths = [str(file.path) for file in file_list] file_paths = [str(file.path) for file in file_list]
try: self.base_url: str | None = self.base_url.strip() if self.base_url else None
parsed_url = urlparse(self.base_url) if self.base_url:
if not parsed_url.hostname or not parsed_url.port: try:
err_msg = "Invalid URL: Missing hostname or port." urlparse(self.base_url)
self.log(err_msg) except Exception as e:
raise ValueError(err_msg) error_msg = f"Invalid Base URL format: {e}"
except Exception as e: self.log(error_msg)
self.log(f"Error parsing URL: {e}") raise ValueError(error_msg) from e
raise else:
base_url_error = "Base URL is required"
raise ValueError(base_url_error)
self.log( self.log(
f"Creating Ingestor for host: {parsed_url.hostname!r}, port: {parsed_url.port!r}", f"Creating Ingestor for Base URL: {self.base_url!r}",
) )
try:
from nv_ingest_client.client import Ingestor
try:
ingestor = ( ingestor = (
Ingestor(message_client_hostname=parsed_url.hostname, message_client_port=parsed_url.port) Ingestor(
message_client_kwargs={
"base_url": self.base_url,
"headers": {"Authorization": f"Bearer {self.api_key}"},
"max_retries": 3,
"timeout": 60,
}
)
.files(file_paths) .files(file_paths)
.extract( .extract(
extract_text=self.extract_text, extract_text=self.extract_text,
extract_tables=self.extract_tables, extract_tables=self.extract_tables,
extract_charts=self.extract_charts, extract_charts=self.extract_charts,
extract_images=False, # Currently not supported extract_images=self.extract_images,
text_depth=self.text_depth, text_depth=self.text_depth,
) )
) )
except Exception as e:
self.log(f"Error creating Ingestor: {e}")
raise
if self.split_text: if self.extract_images:
ingestor = ingestor.split( if self.dedup_images:
split_by=self.split_by, ingestor = ingestor.dedup(content_type="image", filter=True)
split_length=self.split_length,
split_overlap=self.split_overlap, if self.filter_images:
max_character_length=self.max_character_length, ingestor = ingestor.filter(
sentence_window_size=self.sentence_window_size, content_type="image",
) min_size=self.min_image_size,
min_aspect_ratio=self.min_aspect_ratio,
max_aspect_ratio=self.max_aspect_ratio,
filter=True,
)
if self.caption_images:
ingestor = ingestor.caption()
if self.extract_text and self.split_text:
ingestor = ingestor.split(
tokenizer="intfloat/e5-large-unsupervised",
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
params={"split_source_types": ["PDF"]},
)
try:
result = ingestor.ingest() result = ingestor.ingest()
except Exception as e: except Exception as e:
self.log(f"Error during ingestion: {e}") ingest_error = f"Error during ingestion: {e}"
self.log(ingest_error)
raise raise
self.log(f"Results: {result}") self.log(f"Results: {result}")
@ -181,37 +239,55 @@ class NvidiaIngestComponent(BaseFileComponent):
# Result is a list of segments as determined by the text_depth option (if "document" then only one segment) # Result is a list of segments as determined by the text_depth option (if "document" then only one segment)
# each segment is a list of elements (text, structured, image) # each segment is a list of elements (text, structured, image)
for segment in result: for segment in result:
for element in segment: if segment:
document_type = element.get("document_type") for element in segment:
metadata = element.get("metadata", {}) document_type = element.get("document_type")
source_metadata = metadata.get("source_metadata", {}) metadata = element.get("metadata", {})
content_metadata = metadata.get("content_metadata", {}) source_metadata = metadata.get("source_metadata", {})
if document_type == document_type_text: if document_type == document_type_text:
data.append( data.append(
Data( Data(
text=metadata.get("content", ""), text=metadata.get("content", ""),
file_path=source_metadata.get("source_name", ""), file_path=source_metadata.get("source_name", ""),
document_type=document_type, document_type=document_type,
description=content_metadata.get("description", ""), metadata=metadata,
)
) )
) # Both charts and tables are returned as "structured" document type,
# Both charts and tables are returned as "structured" document type, # with extracted text in "table_content"
# with extracted text in "table_content" elif document_type == document_type_structured:
elif document_type == document_type_structured: table_metadata = metadata.get("table_metadata", {})
table_metadata = metadata.get("table_metadata", {})
data.append(
Data(
text=table_metadata.get("table_content", ""),
file_path=source_metadata.get("source_name", ""),
document_type=document_type,
description=content_metadata.get("description", ""),
)
)
else:
# image is not yet supported; skip if encountered
self.log(f"Unsupported document type: {document_type}")
# reformat chart/table images as binary data
if "content" in metadata:
metadata["content"] = {"$binary": metadata["content"]}
data.append(
Data(
text=table_metadata.get("table_content", ""),
file_path=source_metadata.get("source_name", ""),
document_type=document_type,
metadata=metadata,
)
)
elif document_type == "image":
image_metadata = metadata.get("image_metadata", {})
# reformat images as binary data
if "content" in metadata:
metadata["content"] = {"$binary": metadata["content"]}
data.append(
Data(
text=image_metadata.get("caption", "No caption available"),
file_path=source_metadata.get("source_name", ""),
document_type=document_type,
metadata=metadata,
)
)
else:
self.log(f"Unsupported document type {document_type}")
self.status = data or "No data" self.status = data or "No data"
# merge processed data with BaseFile objects # merge processed data with BaseFile objects

8631
uv.lock generated
View file

File diff suppressed because it is too large Load diff