feat: enhance URL component with recursive crawling (#7006)
* feat: Enhance URL component with recursive crawling and advanced options. - Implement advanced URL loading with configurable depth, domain prevention, and async options. * chore: move component to a new file and restore the old one * test: add test to new component * [autofix.ci] apply automated fixes * feat: enhance URLComponent for recursive URL loading - Updated URLComponent to support recursive loading and parsing of child links from a root URL. - Introduced new inputs: max_depth for controlling link traversal depth and prevent_outside to restrict crawling to the same domain. - Replaced AsyncHtmlLoader with RecursiveUrlLoader for improved document loading. - Enhanced error handling and logging for better debugging. - Updated output methods to return structured data and text content more effectively. * chore: delete component * chore: update component tests * ✨ (freeze.spec.ts): refactor click actions to improve readability and maintainability ♻️ (loop-component.spec.ts): refactor click actions to improve readability and maintainability 🔧 (chatInputOutputUser-shard-1.spec.ts): add click action to fit view element for better user experience * 🐛 (url.py): fix issue where data was being returned before error handling, causing potential errors to be missed 🐛 (freeze.spec.ts): fix incorrect test assertions for text comparison, ensuring correct comparison logic is applied * [autofix.ci] apply automated fixes * fix: ruff error * ✨ (stop-building.spec.ts): refactor test to use click method instead of hover and mouse events for better readability and reliability ✨ (loop-component.spec.ts): update test to use consistent naming convention for handle-urlcomponent-shownode-data-right ✨ (chatInputOutputUser-shard-1.spec.ts): update test to use consistent naming convention for handle-urlcomponent-shownode-message-right and handle-urlcomponent-shownode-dataframe-right * ✨ (chatInputOutputUser-shard-1.spec.ts): add additional wait time before running and verifying data output to ensure proper loading and display of data * fix: playwrite test * fix: ruff error * ✅ (auto-login-off.spec.ts): add delay before continuing test to ensure proper execution ✅ (freeze-path.spec.ts): increase timeout for certain actions to prevent test failures due to timing issues ✅ (freeze-path.spec.ts): add delay before continuing test to ensure proper execution ✅ (dropdownComponent.spec.ts): refactor test to use a more reliable assertion for dropdown value length * ✨ (freeze.spec.ts): add additional wait time before clicking on a button to improve test stability ✨ (freeze.spec.ts): increase timeout for waiting for "built successfully" text to appear to improve test reliability ✨ (globalVariables.spec.ts): add initial setup function to skip certain steps and improve test efficiency --------- Co-authored-by: italojohnny <italojohnnydosanjos@gmail.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Edwin Jose <edwin.jose@datastax.com> Co-authored-by: cristhianzl <cristhian.lousa@gmail.com>
This commit is contained in:
Rodrigo Nader
GitHub
parent
59b2ed7765
commit
35d62a033f
9 changed files with 307 additions and 334 deletions
|
|
@ -21,6 +21,9 @@ class TestURLComponent(ComponentTestBaseWithoutClient):
|
|||
return {
|
||||
"urls": ["https://google.com"],
|
||||
"format": "Text",
|
||||
"max_depth": 1,
|
||||
"prevent_outside": True,
|
||||
"use_async": True,
|
||||
}
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -30,55 +33,37 @@ class TestURLComponent(ComponentTestBaseWithoutClient):
|
|||
{"version": "1.0.19", "module": "data", "file_name": "URL"},
|
||||
{"version": "1.1.0", "module": "data", "file_name": "url"},
|
||||
{"version": "1.1.1", "module": "data", "file_name": "url"},
|
||||
{"version": "1.2.0", "module": "data", "file_name": "url"},
|
||||
]
|
||||
|
||||
@pytest.fixture
|
||||
def mock_web_load(self):
|
||||
"""Mock the WebBaseLoader.load method."""
|
||||
with patch("langchain_community.document_loaders.WebBaseLoader.load") as mock:
|
||||
def mock_recursive_loader(self):
|
||||
"""Mock the RecursiveUrlLoader.load method."""
|
||||
with patch("langchain_community.document_loaders.RecursiveUrlLoader.load") as mock:
|
||||
yield mock
|
||||
|
||||
def test_url_component(self, mock_web_load):
|
||||
"""Test basic URL component functionality."""
|
||||
def test_recursive_url_component(self, mock_recursive_loader):
|
||||
"""Test basic URLComponent functionality."""
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": ["https://example.com"]})
|
||||
component.set_attributes({"urls": ["https://example.com"], "max_depth": 2})
|
||||
|
||||
mock_web_load.return_value = [Mock(page_content="test content", metadata={"source": "https://example.com"})]
|
||||
mock_recursive_loader.return_value = [
|
||||
Mock(page_content="test content", metadata={"source": "https://example.com"})
|
||||
]
|
||||
|
||||
data_ = component.fetch_content()
|
||||
assert all(value.data for value in data_)
|
||||
assert all(value.text for value in data_)
|
||||
assert all(value.source for value in data_)
|
||||
|
||||
# @pytest.mark.parametrize(
|
||||
# ("format_type", "expected_content"),
|
||||
# [
|
||||
# ("Text", "test content"),
|
||||
# ("Raw HTML", "<html>test content</html>"),
|
||||
# ],
|
||||
# )
|
||||
# def test_url_component_formats(self, mock_web_load, format_type, expected_content):
|
||||
# """Test URL component with different format types."""
|
||||
# component = URLComponent()
|
||||
# component.set_attributes({"urls": ["https://example.com"], "format": format_type})
|
||||
|
||||
# # Mock the loader response
|
||||
# mock_web_load.return_value = [Mock(page_content=expected_content, metadata={"source": "https://example.com"})]
|
||||
|
||||
# # Test fetch_content - use sync version
|
||||
# content = component.fetch_content()
|
||||
# assert len(content) == 1
|
||||
# assert content[0].text == expected_content
|
||||
# assert content[0].source == "https://example.com"
|
||||
|
||||
def test_url_component_as_dataframe(self, mock_web_load):
|
||||
"""Test URL component's as_dataframe method."""
|
||||
def test_recursive_url_component_as_dataframe(self, mock_recursive_loader):
|
||||
"""Test URLComponent's as_dataframe method."""
|
||||
component = URLComponent()
|
||||
urls = ["https://example1.com", "https://example2.com"]
|
||||
component.set_attributes({"urls": urls})
|
||||
component.set_attributes({"urls": urls, "max_depth": 1})
|
||||
|
||||
# Mock the loader response
|
||||
mock_web_load.return_value = [
|
||||
mock_recursive_loader.return_value = [
|
||||
Mock(page_content="content1", metadata={"source": urls[0]}),
|
||||
Mock(page_content="content2", metadata={"source": urls[1]}),
|
||||
]
|
||||
|
|
@ -86,42 +71,59 @@ class TestURLComponent(ComponentTestBaseWithoutClient):
|
|||
# Test as_dataframe
|
||||
data_frame = component.as_dataframe()
|
||||
assert isinstance(data_frame, DataFrame), "Expected DataFrame instance"
|
||||
assert len(data_frame) == 2
|
||||
assert len(data_frame) == 4
|
||||
|
||||
assert list(data_frame.columns) == ["text", "source"]
|
||||
|
||||
assert data_frame.iloc[0]["text"] == "content1"
|
||||
assert data_frame.iloc[0]["source"] == urls[0]
|
||||
|
||||
assert data_frame.iloc[1]["text"] == "content2"
|
||||
assert data_frame.iloc[1]["source"] == urls[1]
|
||||
|
||||
def test_url_component_fetch_content_text(self, mock_web_load):
|
||||
"""Test URL component's fetch_content_text method."""
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": ["https://example.com"]})
|
||||
assert data_frame.iloc[2]["text"] == "content1"
|
||||
assert data_frame.iloc[2]["source"] == urls[0]
|
||||
|
||||
mock_web_load.return_value = [Mock(page_content="test content", metadata={"source": "https://example.com"})]
|
||||
assert data_frame.iloc[3]["text"] == "content2"
|
||||
assert data_frame.iloc[3]["source"] == urls[1]
|
||||
|
||||
def test_recursive_url_component_fetch_content_text(self, mock_recursive_loader):
|
||||
"""Test URLComponent's fetch_content_text method."""
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": ["https://example.com"], "max_depth": 1})
|
||||
|
||||
mock_recursive_loader.return_value = [
|
||||
Mock(page_content="test content", metadata={"source": "https://example.com"})
|
||||
]
|
||||
|
||||
# Test fetch_content_text
|
||||
message = component.fetch_content_text()
|
||||
assert isinstance(message, Message), "Expected Message instance"
|
||||
assert message.text == "test content"
|
||||
|
||||
def test_url_component_invalid_urls(self):
|
||||
"""Test URL component with invalid URLs."""
|
||||
def test_recursive_url_component_ensure_url(self):
|
||||
"""Test URLComponent's ensure_url method."""
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": ["not_a_valid_url"]})
|
||||
|
||||
# Test that invalid URLs raise a ValueError
|
||||
with pytest.raises(ValueError, match="Invalid URL - http://not_a_valid_url"):
|
||||
component.fetch_content()
|
||||
# Test URL without protocol
|
||||
url = "example.com"
|
||||
fixed_url = component.ensure_url(url)
|
||||
assert fixed_url == "http://example.com"
|
||||
|
||||
def test_url_component_multiple_urls(self, mock_web_load):
|
||||
"""Test URL component with multiple URLs."""
|
||||
# Test URL with protocol
|
||||
url = "http://example.com"
|
||||
fixed_url = component.ensure_url(url)
|
||||
assert fixed_url == "http://example.com"
|
||||
|
||||
def test_recursive_url_component_multiple_urls(self, mock_recursive_loader):
|
||||
"""Test URLComponent with multiple URLs."""
|
||||
component = URLComponent()
|
||||
urls = ["https://example1.com", "https://example2.com", "https://example3.com"]
|
||||
component.set_attributes({"urls": urls})
|
||||
component.set_attributes({"urls": urls, "max_depth": 1})
|
||||
|
||||
mock_web_load.return_value = [
|
||||
Mock(page_content=f"content{i + 1}", metadata={"source": url}) for i, url in enumerate(urls)
|
||||
# Mock different content for each URL
|
||||
mock_recursive_loader.side_effect = [
|
||||
[Mock(page_content=f"content{i + 1}", metadata={"source": url})] for i, url in enumerate(urls)
|
||||
]
|
||||
|
||||
# Test fetch_content
|
||||
|
|
@ -129,20 +131,52 @@ class TestURLComponent(ComponentTestBaseWithoutClient):
|
|||
assert len(content) == 3, f"Expected 3 content items, got {len(content)}"
|
||||
|
||||
for i, item in enumerate(content):
|
||||
url = urls[i]
|
||||
assert item.source == url, f"Expected '{url}', got '{item.source}'"
|
||||
assert item.source == urls[i], f"Expected '{urls[i]}', got '{item.source}'"
|
||||
assert item.text == f"content{i + 1}"
|
||||
|
||||
@patch("langflow.components.data.URLComponent.ensure_url")
|
||||
def test_recursive_url_component_error_handling(self, mock_recursive_loader):
|
||||
"""Test error handling in URLComponent."""
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": ["https://example.com"]})
|
||||
|
||||
# Set up the mock to raise an exception
|
||||
mock_recursive_loader.side_effect = Exception("Connection error")
|
||||
|
||||
# Test that exceptions are properly handled
|
||||
with pytest.raises(ValueError, match="Error loading documents: Connection error"):
|
||||
component.fetch_content()
|
||||
|
||||
def test_recursive_url_component_format_options(self, mock_recursive_loader):
|
||||
"""Test URLComponent with different format options."""
|
||||
component = URLComponent()
|
||||
|
||||
# Test with Text format
|
||||
component.set_attributes({"urls": ["https://example.com"], "format": "Text"})
|
||||
mock_recursive_loader.return_value = [
|
||||
Mock(page_content="extracted text", metadata={"source": "https://example.com"})
|
||||
]
|
||||
content_text = component.fetch_content()
|
||||
assert content_text[0].text == "extracted text"
|
||||
|
||||
# Test with Raw HTML format
|
||||
component.set_attributes({"urls": ["https://example.com"], "format": "Raw HTML"})
|
||||
mock_recursive_loader.return_value = [
|
||||
Mock(page_content="<html>raw html</html>", metadata={"source": "https://example.com"})
|
||||
]
|
||||
content_html = component.fetch_content()
|
||||
assert content_html[0].text == "<html>raw html</html>"
|
||||
|
||||
@respx.mock
|
||||
async def test_url_request_success(self, mock_web_load):
|
||||
async def test_url_request_success(self, mock_recursive_loader):
|
||||
"""Test successful URL request."""
|
||||
url = "https://example.com/api/test"
|
||||
respx.get(url).mock(return_value=Response(200, json={"success": True}))
|
||||
|
||||
component = URLComponent()
|
||||
component.set_attributes({"urls": [url]})
|
||||
component.set_attributes({"urls": [url], "max_depth": 1})
|
||||
|
||||
mock_web_load.return_value = [Mock(page_content="test content", metadata={"source": url})]
|
||||
mock_recursive_loader.return_value = [Mock(page_content="test content", metadata={"source": url})]
|
||||
|
||||
result = component.fetch_content()
|
||||
assert len(result) == 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue