feat: Add not contains filter operator in DataFrame Operations Component (#9415)
* 🐛 (dataframe_operations.py): Fix bug in DataFrameOperationsComponent where "not contains" filter option was missing, causing incorrect filtering behavior. * [autofix.ci] apply automated fixes * Update pyproject versions * fix: Avoid namespace collision for Astra (#9544) * fix: Avoid namespace collision for Astra * [autofix.ci] apply automated fixes * Update Vector Store RAG.json * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> * fix: Revert to a working composio release for module import (#9569) fix: revert to stable composio version * fix: Knowledge base component refactor (#9543) * fix: Knowledge base component refactor * [autofix.ci] apply automated fixes * [autofix.ci] apply automated fixes (attempt 2/3) * Update styleUtils.ts * Update ingestion.py * [autofix.ci] apply automated fixes * Fix ingestion of df * [autofix.ci] apply automated fixes * Update Knowledge Ingestion.json * Fix one failing test * [autofix.ci] apply automated fixes * [autofix.ci] apply automated fixes * Revert composio versions for CI * Revert "Revert composio versions for CI" This reverts commit 9bcb694ea1e20d544cf5e17fed2f9f4c0a3c192b. --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Edwin Jose <edwin.jose@datastax.com> Co-authored-by: Carlos Coelho <80289056+carlosrcoelho@users.noreply.github.com> * fix: Fix env file handling in Windows build scripts (#9414) fix .env load on windows script Co-authored-by: Ítalo Johnny <italojohnnydosanjos@gmail.com> * fix: update agent_llm display name to "Model Provider" in AgentComponent (#9564) Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> * 📝 (test_mcp_util.py): add a check to skip test if DeepWiki server is rate limiting requests to avoid false test failures * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Jordan Frazier <jordan.frazier@datastax.com> Co-authored-by: Eric Hare <ericrhare@gmail.com> Co-authored-by: Edwin Jose <edwin.jose@datastax.com> Co-authored-by: Carlos Coelho <80289056+carlosrcoelho@users.noreply.github.com> Co-authored-by: Ítalo Johnny <italojohnnydosanjos@gmail.com>
This commit is contained in:
Cristhian Zanforlin Lousa
GitHub
parent
1716976994
commit
ab017bafcd
41 changed files with 3041 additions and 2970 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "langflow"
|
||||
version = "1.5.0.post2"
|
||||
version = "1.6.0"
|
||||
description = "A Python package with a built-in web application"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
@echo off
|
||||
echo Starting Langflow build and run process...
|
||||
|
||||
REM Check if .env file exists and set env file parameter
|
||||
set "ENV_FILE_PARAM="
|
||||
REM Check if .env file exists and set env file flag
|
||||
set "USE_ENV_FILE="
|
||||
REM Get the script directory and resolve project root
|
||||
for %%I in ("%~dp0..\..") do set "PROJECT_ROOT=%%~fI"
|
||||
set "ENV_PATH=%PROJECT_ROOT%\.env"
|
||||
if exist "%ENV_PATH%" (
|
||||
echo Found .env file at: %ENV_PATH%
|
||||
set "ENV_FILE_PARAM=--env-file \"%ENV_PATH%\""
|
||||
set "USE_ENV_FILE=1"
|
||||
) else (
|
||||
echo .env file not found at: %ENV_PATH%
|
||||
echo Langflow will use default configuration
|
||||
|
|
@ -85,8 +85,8 @@ echo Step 4: Running Langflow...
|
|||
echo.
|
||||
echo Attention: Wait until uvicorn is running before opening the browser
|
||||
echo.
|
||||
if defined ENV_FILE_PARAM (
|
||||
uv run langflow run %ENV_FILE_PARAM%
|
||||
if defined USE_ENV_FILE (
|
||||
uv run --env-file "%ENV_PATH%" langflow run
|
||||
) else (
|
||||
uv run langflow run
|
||||
)
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ Write-Host "`nStep 4: Running Langflow..." -ForegroundColor Yellow
|
|||
Write-Host "`nAttention: Wait until uvicorn is running before opening the browser" -ForegroundColor Red
|
||||
try {
|
||||
if ($useEnvFile) {
|
||||
& uv run langflow run --env-file $envPath
|
||||
& uv run --env-file $envPath langflow run
|
||||
} else {
|
||||
& uv run langflow run
|
||||
}
|
||||
|
|
|
|||
|
|
@ -483,6 +483,7 @@ class AgentComponent(ToolCallingAgentComponent):
|
|||
build_config.update(fields_to_add)
|
||||
# Reset input types for agent_llm
|
||||
build_config["agent_llm"]["input_types"] = []
|
||||
build_config["agent_llm"]["display_name"] = "Model Provider"
|
||||
elif field_value == "Custom":
|
||||
# Delete all provider fields
|
||||
self.delete_fields(build_config, ALL_PROVIDER_FIELDS)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,6 @@ from .csv_to_data import CSVToDataComponent
|
|||
from .directory import DirectoryComponent
|
||||
from .file import FileComponent
|
||||
from .json_to_data import JSONToDataComponent
|
||||
from .kb_ingest import KBIngestionComponent
|
||||
from .kb_retrieval import KBRetrievalComponent
|
||||
from .news_search import NewsSearchComponent
|
||||
from .rss import RSSReaderComponent
|
||||
from .sql_executor import SQLComponent
|
||||
|
|
@ -18,8 +16,6 @@ __all__ = [
|
|||
"DirectoryComponent",
|
||||
"FileComponent",
|
||||
"JSONToDataComponent",
|
||||
"KBIngestionComponent",
|
||||
"KBRetrievalComponent",
|
||||
"NewsSearchComponent",
|
||||
"RSSReaderComponent",
|
||||
"SQLComponent",
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@ if TYPE_CHECKING:
|
|||
from .astra_assistant_manager import AstraAssistantManager
|
||||
from .astra_db import AstraDBChatMemory
|
||||
from .astra_vectorize import AstraVectorizeComponent
|
||||
from .astradb import AstraDBVectorStoreComponent
|
||||
from .astradb_cql import AstraDBCQLToolComponent
|
||||
from .astradb_tool import AstraDBToolComponent
|
||||
from .astradb_vectorstore import AstraDBVectorStoreComponent
|
||||
from .create_assistant import AssistantsCreateAssistant
|
||||
from .create_thread import AssistantsCreateThread
|
||||
from .dotenv import Dotenv
|
||||
|
|
@ -29,7 +29,7 @@ _dynamic_imports = {
|
|||
"AstraDBCQLToolComponent": "astradb_cql",
|
||||
"AstraDBChatMemory": "astra_db",
|
||||
"AstraDBToolComponent": "astradb_tool",
|
||||
"AstraDBVectorStoreComponent": "astradb",
|
||||
"AstraDBVectorStoreComponent": "astradb_vectorstore",
|
||||
"AstraVectorizeComponent": "astra_vectorize",
|
||||
"Dotenv": "dotenv",
|
||||
"GetEnvVar": "getenvvar",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from langflow.components._importing import import_mod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langflow.components.knowledge_bases.ingestion import KnowledgeIngestionComponent
|
||||
from langflow.components.knowledge_bases.retrieval import KnowledgeRetrievalComponent
|
||||
|
||||
_dynamic_imports = {
|
||||
"KnowledgeIngestionComponent": "ingestion",
|
||||
"KnowledgeRetrievalComponent": "retrieval",
|
||||
}
|
||||
|
||||
__all__ = ["KnowledgeIngestionComponent", "KnowledgeRetrievalComponent"]
|
||||
|
||||
|
||||
def __getattr__(attr_name: str) -> Any:
|
||||
"""Lazily import input/output components on attribute access."""
|
||||
if attr_name not in _dynamic_imports:
|
||||
msg = f"module '{__name__}' has no attribute '{attr_name}'"
|
||||
raise AttributeError(msg)
|
||||
try:
|
||||
result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent)
|
||||
except (ModuleNotFoundError, ImportError, AttributeError) as e:
|
||||
msg = f"Could not import '{attr_name}' from '{__name__}': {e}"
|
||||
raise AttributeError(msg) from e
|
||||
globals()[attr_name] = result
|
||||
return result
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
return list(__all__)
|
||||
|
|
@ -9,17 +9,18 @@ import uuid
|
|||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pandas as pd
|
||||
from cryptography.fernet import InvalidToken
|
||||
from langchain_chroma import Chroma
|
||||
from loguru import logger
|
||||
|
||||
from langflow.base.data.kb_utils import get_knowledge_bases
|
||||
from langflow.base.knowledge_bases.knowledge_base_utils import get_knowledge_bases
|
||||
from langflow.base.models.openai_constants import OPENAI_EMBEDDING_MODEL_NAMES
|
||||
from langflow.components.processing.converter import convert_to_dataframe
|
||||
from langflow.custom import Component
|
||||
from langflow.io import BoolInput, DataFrameInput, DropdownInput, IntInput, Output, SecretStrInput, StrInput, TableInput
|
||||
from langflow.io import BoolInput, DropdownInput, HandleInput, IntInput, Output, SecretStrInput, StrInput, TableInput
|
||||
from langflow.schema.data import Data
|
||||
from langflow.schema.dotdict import dotdict # noqa: TC001
|
||||
from langflow.schema.table import EditMode
|
||||
|
|
@ -27,6 +28,9 @@ from langflow.services.auth.utils import decrypt_api_key, encrypt_api_key
|
|||
from langflow.services.database.models.user.crud import get_user_by_id
|
||||
from langflow.services.deps import get_settings_service, get_variable_service, session_scope
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langflow.schema.dataframe import DataFrame
|
||||
|
||||
HUGGINGFACE_MODEL_NAMES = ["sentence-transformers/all-MiniLM-L6-v2", "sentence-transformers/all-mpnet-base-v2"]
|
||||
COHERE_MODEL_NAMES = ["embed-english-v3.0", "embed-multilingual-v3.0"]
|
||||
|
||||
|
|
@ -38,14 +42,14 @@ if not knowledge_directory:
|
|||
KNOWLEDGE_BASES_ROOT_PATH = Path(knowledge_directory).expanduser()
|
||||
|
||||
|
||||
class KBIngestionComponent(Component):
|
||||
class KnowledgeIngestionComponent(Component):
|
||||
"""Create or append to Langflow Knowledge from a DataFrame."""
|
||||
|
||||
# ------ UI metadata ---------------------------------------------------
|
||||
display_name = "Knowledge Ingestion"
|
||||
description = "Create or update knowledge in Langflow."
|
||||
icon = "database"
|
||||
name = "KBIngestion"
|
||||
icon = "upload"
|
||||
name = "KnowledgeIngestion"
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -101,12 +105,17 @@ class KBIngestionComponent(Component):
|
|||
required=True,
|
||||
options=[],
|
||||
refresh_button=True,
|
||||
real_time_refresh=True,
|
||||
dialog_inputs=asdict(NewKnowledgeBaseInput()),
|
||||
),
|
||||
DataFrameInput(
|
||||
HandleInput(
|
||||
name="input_df",
|
||||
display_name="Data",
|
||||
info="Table with all original columns (already chunked / processed).",
|
||||
display_name="Input",
|
||||
info=(
|
||||
"Table with all original columns (already chunked / processed). "
|
||||
"Accepts Data or DataFrame. If Data is provided, it is converted to a DataFrame automatically."
|
||||
),
|
||||
input_types=["Data", "DataFrame"],
|
||||
required=True,
|
||||
),
|
||||
TableInput(
|
||||
|
|
@ -171,7 +180,7 @@ class KBIngestionComponent(Component):
|
|||
]
|
||||
|
||||
# ------ Outputs -------------------------------------------------------
|
||||
outputs = [Output(display_name="DataFrame", name="dataframe", method="build_kb_info")]
|
||||
outputs = [Output(display_name="Results", name="dataframe_output", method="build_kb_info")]
|
||||
|
||||
# ------ Internal helpers ---------------------------------------------
|
||||
def _get_kb_root(self) -> Path:
|
||||
|
|
@ -503,8 +512,8 @@ class KBIngestionComponent(Component):
|
|||
async def build_kb_info(self) -> Data:
|
||||
"""Main ingestion routine → returns a dict with KB metadata."""
|
||||
try:
|
||||
# Get source DataFrame
|
||||
df_source: pd.DataFrame = self.input_df
|
||||
input_value = self.input_df[0] if isinstance(self.input_df, list) else self.input_df
|
||||
df_source: DataFrame = convert_to_dataframe(input_value)
|
||||
|
||||
# Validate column configuration (using Structured Output patterns)
|
||||
config_list = self._validate_column_config(df_source)
|
||||
|
|
@ -559,9 +568,8 @@ class KBIngestionComponent(Component):
|
|||
return Data(data=meta)
|
||||
|
||||
except (OSError, ValueError, RuntimeError, KeyError) as e:
|
||||
self.log(f"Error in KB ingestion: {e}")
|
||||
self.status = f"❌ KB ingestion failed: {e}"
|
||||
return Data(data={"error": str(e), "kb_name": self.knowledge_base})
|
||||
msg = f"Error during KB ingestion: {e}"
|
||||
raise RuntimeError(msg) from e
|
||||
|
||||
async def _get_api_key_variable(self, field_value: dict[str, Any]):
|
||||
async with session_scope() as db:
|
||||
|
|
@ -7,7 +7,7 @@ from langchain_chroma import Chroma
|
|||
from loguru import logger
|
||||
from pydantic import SecretStr
|
||||
|
||||
from langflow.base.data.kb_utils import get_knowledge_bases
|
||||
from langflow.base.knowledge_bases.knowledge_base_utils import get_knowledge_bases
|
||||
from langflow.custom import Component
|
||||
from langflow.io import BoolInput, DropdownInput, IntInput, MessageTextInput, Output, SecretStrInput
|
||||
from langflow.schema.data import Data
|
||||
|
|
@ -24,11 +24,11 @@ if not knowledge_directory:
|
|||
KNOWLEDGE_BASES_ROOT_PATH = Path(knowledge_directory).expanduser()
|
||||
|
||||
|
||||
class KBRetrievalComponent(Component):
|
||||
class KnowledgeRetrievalComponent(Component):
|
||||
display_name = "Knowledge Retrieval"
|
||||
description = "Search and retrieve data from knowledge."
|
||||
icon = "database"
|
||||
name = "KBRetrieval"
|
||||
icon = "download"
|
||||
name = "KnowledgeRetrieval"
|
||||
|
||||
inputs = [
|
||||
DropdownInput(
|
||||
|
|
@ -51,6 +51,7 @@ class KBRetrievalComponent(Component):
|
|||
name="search_query",
|
||||
display_name="Search Query",
|
||||
info="Optional search query to filter knowledge base data.",
|
||||
tool_mode=True,
|
||||
),
|
||||
IntInput(
|
||||
name="top_k",
|
||||
|
|
@ -63,17 +64,24 @@ class KBRetrievalComponent(Component):
|
|||
BoolInput(
|
||||
name="include_metadata",
|
||||
display_name="Include Metadata",
|
||||
info="Whether to include all metadata and embeddings in the output. If false, only content is returned.",
|
||||
info="Whether to include all metadata in the output. If false, only content is returned.",
|
||||
value=True,
|
||||
advanced=False,
|
||||
),
|
||||
BoolInput(
|
||||
name="include_embeddings",
|
||||
display_name="Include Embeddings",
|
||||
info="Whether to include embeddings in the output. Only applicable if 'Include Metadata' is enabled.",
|
||||
value=False,
|
||||
advanced=True,
|
||||
),
|
||||
]
|
||||
|
||||
outputs = [
|
||||
Output(
|
||||
name="chroma_kb_data",
|
||||
name="retrieve_data",
|
||||
display_name="Results",
|
||||
method="get_chroma_kb_data",
|
||||
method="retrieve_data",
|
||||
info="Returns the data from the selected knowledge base.",
|
||||
),
|
||||
]
|
||||
|
|
@ -162,7 +170,7 @@ class KBRetrievalComponent(Component):
|
|||
msg = f"Embedding provider '{provider}' is not supported for retrieval."
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
async def get_chroma_kb_data(self) -> DataFrame:
|
||||
async def retrieve_data(self) -> DataFrame:
|
||||
"""Retrieve data from the selected knowledge base by reading the Chroma collection.
|
||||
|
||||
Returns:
|
||||
|
|
@ -212,16 +220,16 @@ class KBRetrievalComponent(Component):
|
|||
# For each result, make it a tuple to match the expected output format
|
||||
results = [(doc, 0) for doc in results] # Assign a dummy score of 0
|
||||
|
||||
# If metadata is enabled, get embeddings for the results
|
||||
# If include_embeddings is enabled, get embeddings for the results
|
||||
id_to_embedding = {}
|
||||
if self.include_metadata and results:
|
||||
if self.include_embeddings and results:
|
||||
doc_ids = [doc[0].metadata.get("_id") for doc in results if doc[0].metadata.get("_id")]
|
||||
|
||||
# Only proceed if we have valid document IDs
|
||||
if doc_ids:
|
||||
# Access underlying client to get embeddings
|
||||
collection = chroma._client.get_collection(name=self.knowledge_base)
|
||||
embeddings_result = collection.get(where={"_id": {"$in": doc_ids}}, include=["embeddings", "metadatas"])
|
||||
embeddings_result = collection.get(where={"_id": {"$in": doc_ids}}, include=["metadatas", "embeddings"])
|
||||
|
||||
# Create a mapping from document ID to embedding
|
||||
for i, metadata in enumerate(embeddings_result.get("metadatas", [])):
|
||||
|
|
@ -231,20 +239,16 @@ class KBRetrievalComponent(Component):
|
|||
# Build output data based on include_metadata setting
|
||||
data_list = []
|
||||
for doc in results:
|
||||
kwargs = {
|
||||
"content": doc[0].page_content,
|
||||
}
|
||||
if self.search_query:
|
||||
kwargs["_score"] = -1 * doc[1]
|
||||
if self.include_metadata:
|
||||
# Include all metadata, embeddings, and content
|
||||
kwargs = {
|
||||
"content": doc[0].page_content,
|
||||
**doc[0].metadata,
|
||||
}
|
||||
if self.search_query:
|
||||
kwargs["_score"] = -1 * doc[1]
|
||||
kwargs.update(doc[0].metadata)
|
||||
if self.include_embeddings:
|
||||
kwargs["_embeddings"] = id_to_embedding.get(doc[0].metadata.get("_id"))
|
||||
else:
|
||||
# Only include content
|
||||
kwargs = {
|
||||
"content": doc[0].page_content,
|
||||
}
|
||||
|
||||
data_list.append(Data(**kwargs))
|
||||
|
||||
|
|
@ -79,7 +79,16 @@ class DataFrameOperationsComponent(Component):
|
|||
DropdownInput(
|
||||
name="filter_operator",
|
||||
display_name="Filter Operator",
|
||||
options=["equals", "not equals", "contains", "starts with", "ends with", "greater than", "less than"],
|
||||
options=[
|
||||
"equals",
|
||||
"not equals",
|
||||
"contains",
|
||||
"not contains",
|
||||
"starts with",
|
||||
"ends with",
|
||||
"greater than",
|
||||
"less than",
|
||||
],
|
||||
value="equals",
|
||||
info="The operator to apply for filtering rows.",
|
||||
advanced=False,
|
||||
|
|
@ -254,6 +263,8 @@ class DataFrameOperationsComponent(Component):
|
|||
mask = column != filter_value
|
||||
elif operator == "contains":
|
||||
mask = column.astype(str).str.contains(str(filter_value), na=False)
|
||||
elif operator == "not contains":
|
||||
mask = ~column.astype(str).str.contains(str(filter_value), na=False)
|
||||
elif operator == "starts with":
|
||||
mask = column.astype(str).str.startswith(str(filter_value), na=False)
|
||||
elif operator == "ends with":
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "langflow-base"
|
||||
version = "0.5.0.post2"
|
||||
version = "0.6.0"
|
||||
description = "A Python package with a built-in web application"
|
||||
requires-python = ">=3.10,<3.14"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import pytest
|
||||
from langflow.base.data.kb_utils import compute_bm25, compute_tfidf
|
||||
from langflow.base.knowledge_bases.knowledge_base_utils import compute_bm25, compute_tfidf
|
||||
|
||||
|
||||
class TestKBUtils:
|
||||
|
|
|
|||
|
|
@ -594,6 +594,11 @@ class TestMCPSseClientWithDeepWikiServer:
|
|||
# Test valid URL
|
||||
valid_url = "https://mcp.deepwiki.com/sse"
|
||||
is_valid, error = await sse_client.validate_url(valid_url)
|
||||
# Either valid or accessible, or rate-limited (429) which indicates server is reachable
|
||||
if not is_valid and "429" in error:
|
||||
# Rate limiting indicates the server is accessible but limiting requests
|
||||
# This is a transient network issue, not a test failure
|
||||
pytest.skip(f"DeepWiki server is rate limiting requests: {error}")
|
||||
assert is_valid or error == "" # Either valid or accessible
|
||||
|
||||
# Test invalid URL
|
||||
|
|
|
|||
|
|
@ -2,25 +2,25 @@ import json
|
|||
import uuid
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from langflow.base.data.kb_utils import get_knowledge_bases
|
||||
from langflow.components.data.kb_ingest import KBIngestionComponent
|
||||
from langflow.base.knowledge_bases.knowledge_base_utils import get_knowledge_bases
|
||||
from langflow.components.knowledge_bases.ingestion import KnowledgeIngestionComponent
|
||||
from langflow.schema.data import Data
|
||||
from langflow.schema.dataframe import DataFrame
|
||||
|
||||
from tests.base import ComponentTestBaseWithoutClient
|
||||
|
||||
|
||||
class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
||||
class TestKnowledgeIngestionComponent(ComponentTestBaseWithoutClient):
|
||||
@pytest.fixture
|
||||
def component_class(self):
|
||||
"""Return the component class to test."""
|
||||
return KBIngestionComponent
|
||||
return KnowledgeIngestionComponent
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_knowledge_base_path(self, tmp_path):
|
||||
"""Mock the knowledge base root path directly."""
|
||||
with patch("langflow.components.data.kb_ingest.KNOWLEDGE_BASES_ROOT_PATH", tmp_path):
|
||||
with patch("langflow.components.knowledge_bases.ingestion.KNOWLEDGE_BASES_ROOT_PATH", tmp_path):
|
||||
yield
|
||||
|
||||
class MockUser:
|
||||
|
|
@ -39,14 +39,14 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
def setup_mocks(self, mock_user_data):
|
||||
"""Mock the component's user_id attribute and User object."""
|
||||
with (
|
||||
patch.object(KBIngestionComponent, "user_id", mock_user_data["user_id"]),
|
||||
patch.object(KnowledgeIngestionComponent, "user_id", mock_user_data["user_id"]),
|
||||
patch(
|
||||
"langflow.components.data.kb_ingest.get_user_by_id",
|
||||
"langflow.components.knowledge_bases.ingestion.get_user_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_user_data["user_obj"],
|
||||
),
|
||||
patch(
|
||||
"langflow.base.data.kb_utils.get_user_by_id",
|
||||
"langflow.base.knowledge_bases.knowledge_base_utils.get_user_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_user_data["user_obj"],
|
||||
),
|
||||
|
|
@ -62,7 +62,7 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
def default_kwargs(self, tmp_path, mock_user_id):
|
||||
"""Return default kwargs for component instantiation."""
|
||||
# Create a sample DataFrame
|
||||
data_df = pd.DataFrame(
|
||||
data_df = DataFrame(
|
||||
{"text": ["Sample text 1", "Sample text 2"], "title": ["Title 1", "Title 2"], "category": ["cat1", "cat2"]}
|
||||
)
|
||||
|
||||
|
|
@ -209,8 +209,8 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
with pytest.raises(NotImplementedError, match="Custom embedding models not yet supported"):
|
||||
component._build_embeddings("custom-model", "test-key")
|
||||
|
||||
@patch("langflow.components.data.kb_ingest.get_settings_service")
|
||||
@patch("langflow.components.data.kb_ingest.encrypt_api_key")
|
||||
@patch("langflow.components.knowledge_bases.ingestion.get_settings_service")
|
||||
@patch("langflow.components.knowledge_bases.ingestion.encrypt_api_key")
|
||||
def test_build_embedding_metadata(self, mock_encrypt, mock_get_settings, component_class, default_kwargs):
|
||||
"""Test building embedding metadata."""
|
||||
component = component_class(**default_kwargs)
|
||||
|
|
@ -250,7 +250,7 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
config_list = default_kwargs["column_config"]
|
||||
|
||||
# Mock Chroma to avoid actual vector store operations
|
||||
with patch("langflow.components.data.kb_ingest.Chroma") as mock_chroma:
|
||||
with patch("langflow.components.knowledge_bases.ingestion.Chroma") as mock_chroma:
|
||||
mock_chroma_instance = MagicMock()
|
||||
mock_chroma_instance.get.return_value = {"metadatas": []}
|
||||
mock_chroma.return_value = mock_chroma_instance
|
||||
|
|
@ -275,7 +275,7 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
config_list = default_kwargs["column_config"]
|
||||
|
||||
# Mock Chroma with existing hash
|
||||
with patch("langflow.components.data.kb_ingest.Chroma") as mock_chroma:
|
||||
with patch("langflow.components.knowledge_bases.ingestion.Chroma") as mock_chroma:
|
||||
# Simulate existing document with same hash
|
||||
existing_hash = "some_existing_hash"
|
||||
mock_chroma_instance = MagicMock()
|
||||
|
|
@ -283,7 +283,7 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
mock_chroma.return_value = mock_chroma_instance
|
||||
|
||||
# Mock hashlib to return the existing hash for first row
|
||||
with patch("langflow.components.data.kb_ingest.hashlib.sha256") as mock_hash:
|
||||
with patch("langflow.components.knowledge_bases.ingestion.hashlib.sha256") as mock_hash:
|
||||
mock_hash_obj = MagicMock()
|
||||
mock_hash_obj.hexdigest.side_effect = [existing_hash, "different_hash"]
|
||||
mock_hash.return_value = mock_hash_obj
|
||||
|
|
@ -309,8 +309,8 @@ class TestKBIngestionComponent(ComponentTestBaseWithoutClient):
|
|||
assert component.is_valid_collection_name("invalid_") is False # Ends with underscore
|
||||
assert component.is_valid_collection_name("invalid@name") is False # Invalid character
|
||||
|
||||
@patch("langflow.components.data.kb_ingest.json.loads")
|
||||
@patch("langflow.components.data.kb_ingest.decrypt_api_key")
|
||||
@patch("langflow.components.knowledge_bases.ingestion.json.loads")
|
||||
@patch("langflow.components.knowledge_bases.ingestion.decrypt_api_key")
|
||||
async def test_build_kb_info_success(self, mock_decrypt, mock_json_loads, component_class, default_kwargs):
|
||||
"""Test successful KB info building."""
|
||||
component = component_class(**default_kwargs)
|
||||
|
|
@ -5,23 +5,23 @@ from pathlib import Path
|
|||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from langflow.base.data.kb_utils import get_knowledge_bases
|
||||
from langflow.components.data.kb_retrieval import KBRetrievalComponent
|
||||
from langflow.base.knowledge_bases.knowledge_base_utils import get_knowledge_bases
|
||||
from langflow.components.knowledge_bases.retrieval import KnowledgeRetrievalComponent
|
||||
from pydantic import SecretStr
|
||||
|
||||
from tests.base import ComponentTestBaseWithoutClient
|
||||
|
||||
|
||||
class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
||||
class TestKnowledgeRetrievalComponent(ComponentTestBaseWithoutClient):
|
||||
@pytest.fixture
|
||||
def component_class(self):
|
||||
"""Return the component class to test."""
|
||||
return KBRetrievalComponent
|
||||
return KnowledgeRetrievalComponent
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_knowledge_base_path(self, tmp_path):
|
||||
"""Mock the knowledge base root path directly."""
|
||||
with patch("langflow.components.data.kb_retrieval.KNOWLEDGE_BASES_ROOT_PATH", tmp_path):
|
||||
with patch("langflow.components.knowledge_bases.retrieval.KNOWLEDGE_BASES_ROOT_PATH", tmp_path):
|
||||
yield
|
||||
|
||||
class MockUser:
|
||||
|
|
@ -40,14 +40,14 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
def setup_mocks(self, mock_user_data):
|
||||
"""Mock the component's user_id attribute and User object."""
|
||||
with (
|
||||
patch.object(KBRetrievalComponent, "user_id", mock_user_data["user_id"]),
|
||||
patch.object(KnowledgeRetrievalComponent, "user_id", mock_user_data["user_id"]),
|
||||
patch(
|
||||
"langflow.components.data.kb_retrieval.get_user_by_id",
|
||||
"langflow.components.knowledge_bases.retrieval.get_user_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_user_data["user_obj"],
|
||||
),
|
||||
patch(
|
||||
"langflow.base.data.kb_utils.get_user_by_id",
|
||||
"langflow.base.knowledge_bases.knowledge_base_utils.get_user_by_id",
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_user_data["user_obj"],
|
||||
),
|
||||
|
|
@ -138,7 +138,7 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
component = component_class(**default_kwargs)
|
||||
kb_path = Path(default_kwargs["kb_root_path"]) / mock_user_id["user"] / default_kwargs["knowledge_base"]
|
||||
|
||||
with patch("langflow.components.data.kb_retrieval.decrypt_api_key") as mock_decrypt:
|
||||
with patch("langflow.components.knowledge_bases.retrieval.decrypt_api_key") as mock_decrypt:
|
||||
mock_decrypt.return_value = "decrypted_key"
|
||||
|
||||
metadata = component._get_kb_metadata(kb_path)
|
||||
|
|
@ -185,7 +185,7 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
}
|
||||
(kb_path / "embedding_metadata.json").write_text(json.dumps(metadata))
|
||||
|
||||
with patch("langflow.components.data.kb_retrieval.decrypt_api_key") as mock_decrypt:
|
||||
with patch("langflow.components.knowledge_bases.retrieval.decrypt_api_key") as mock_decrypt:
|
||||
mock_decrypt.side_effect = ValueError("Decryption failed")
|
||||
|
||||
result = component._get_kb_metadata(kb_path)
|
||||
|
|
@ -327,7 +327,7 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
chunk_size=1000,
|
||||
)
|
||||
|
||||
async def test_get_chroma_kb_data_no_metadata(self, component_class, default_kwargs, tmp_path, mock_user_id):
|
||||
async def test_retrieve_data_no_metadata(self, component_class, default_kwargs, tmp_path, mock_user_id):
|
||||
"""Test retrieving data when metadata is missing."""
|
||||
# Remove metadata file
|
||||
kb_path = tmp_path / mock_user_id["user"] / default_kwargs["knowledge_base"]
|
||||
|
|
@ -338,10 +338,10 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
component = component_class(**default_kwargs)
|
||||
|
||||
with pytest.raises(ValueError, match="Metadata not found for knowledge base"):
|
||||
await component.get_chroma_kb_data()
|
||||
await component.retrieve_data()
|
||||
|
||||
def test_get_chroma_kb_data_path_construction(self, component_class, default_kwargs):
|
||||
"""Test that get_chroma_kb_data constructs the correct paths."""
|
||||
def test_retrieve_data_path_construction(self, component_class, default_kwargs):
|
||||
"""Test that retrieve_data constructs the correct paths."""
|
||||
component = component_class(**default_kwargs)
|
||||
|
||||
# Test that the component correctly builds the KB path
|
||||
|
|
@ -354,17 +354,17 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
assert expanded_path.exists() # tmp_path should exist
|
||||
|
||||
# Verify method exists with correct parameters
|
||||
assert hasattr(component, "get_chroma_kb_data")
|
||||
assert hasattr(component, "retrieve_data")
|
||||
assert hasattr(component, "search_query")
|
||||
assert hasattr(component, "top_k")
|
||||
assert hasattr(component, "include_embeddings")
|
||||
|
||||
async def test_get_chroma_kb_data_method_exists(self, component_class, default_kwargs):
|
||||
"""Test that get_chroma_kb_data method exists and can be called."""
|
||||
async def test_retrieve_data_method_exists(self, component_class, default_kwargs):
|
||||
"""Test that retrieve_data method exists and can be called."""
|
||||
component = component_class(**default_kwargs)
|
||||
|
||||
# Just verify the method exists and has the right signature
|
||||
assert hasattr(component, "get_chroma_kb_data"), "Component should have get_chroma_kb_data method"
|
||||
assert hasattr(component, "retrieve_data"), "Component should have retrieve_data method"
|
||||
|
||||
# Mock all external calls to avoid integration issues
|
||||
with (
|
||||
|
|
@ -377,7 +377,7 @@ class TestKBRetrievalComponent(ComponentTestBaseWithoutClient):
|
|||
|
||||
# This is a unit test focused on the component's internal logic
|
||||
with contextlib.suppress(Exception):
|
||||
await component.get_chroma_kb_data()
|
||||
await component.retrieve_data()
|
||||
|
||||
# Verify internal methods were called
|
||||
mock_get_metadata.assert_called_once()
|
||||
|
|
@ -211,6 +211,11 @@ export const SIDEBAR_CATEGORIES = [
|
|||
{ display_name: "Agents", name: "agents", icon: "Bot" },
|
||||
{ display_name: "Models", name: "models", icon: "BrainCog" },
|
||||
{ display_name: "Data", name: "data", icon: "Database" },
|
||||
{
|
||||
display_name: "Knowledge Bases",
|
||||
name: "knowledge_bases",
|
||||
icon: "Library",
|
||||
},
|
||||
{ display_name: "Vector Stores", name: "vectorstores", icon: "Layers" },
|
||||
{ display_name: "Processing", name: "processing", icon: "ListFilter" },
|
||||
{ display_name: "Logic", name: "logic", icon: "ArrowRightLeft" },
|
||||
|
|
|
|||
|
|
@ -97,7 +97,9 @@ test(
|
|||
const nodesFromServer = astraStarterProject?.data.nodes.length;
|
||||
|
||||
expect(
|
||||
edges === edgesFromServer || edges === edgesFromServer - 1,
|
||||
edges === edgesFromServer ||
|
||||
edges === edgesFromServer - 1 ||
|
||||
edges === edgesFromServer - 2,
|
||||
).toBeTruthy();
|
||||
expect(nodes).toBe(nodesFromServer);
|
||||
},
|
||||
|
|
|
|||
4
uv.lock
generated
4
uv.lock
generated
|
|
@ -4975,7 +4975,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "langflow"
|
||||
version = "1.5.0.post2"
|
||||
version = "1.6.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiofile" },
|
||||
|
|
@ -5360,7 +5360,7 @@ dev = [
|
|||
|
||||
[[package]]
|
||||
name = "langflow-base"
|
||||
version = "0.5.0.post2"
|
||||
version = "0.6.0"
|
||||
source = { editable = "src/backend/base" }
|
||||
dependencies = [
|
||||
{ name = "aiofile" },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue