feat: Add OpenSearch VectorStore Component with Ingest and Search Capabilities (#3799)
* update: adding opensearch-py dependency * feat: adding OpenSearch icon * feat: adding OpenSearch VectorStore code * fix: removing unused methods and adding hybrid search capabilities using search method * update: poetry lock with recent hash added * fix: removing unused methods and adding hybrid search capabilities using search method * fix: default value of search_input to an empty string * fix: adapting code to match make format script
This commit is contained in:
João
GitHub
parent
e882f08549
commit
b9bcb09e63
7 changed files with 590 additions and 56 deletions
91
poetry.lock
generated
91
poetry.lock
generated
|
|
@ -2011,61 +2011,6 @@ qdrant = ["fastembed", "qdrant-client"]
|
|||
snowflake = ["snowflake-snowpark-python"]
|
||||
weaviate = ["weaviate-client (>=4.6.5,<4.7.0)"]
|
||||
|
||||
[[package]]
|
||||
name = "duckdb"
|
||||
version = "1.1.0"
|
||||
description = "DuckDB in-process database"
|
||||
optional = false
|
||||
python-versions = ">=3.7.0"
|
||||
files = [
|
||||
{file = "duckdb-1.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5e4cbc408e6e41146dea89b9044dae7356e353db0c96b183e5583ee02bc6ae5d"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6370ae27ec8167ccfbefb94f58ad9fdc7bac142399960549d6d367f233189868"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:4e1c3414f7fd01f4810dc8b335deffc91933a159282d65fef11c1286bc0ded04"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6bc2a58689adf5520303c5f68b065b9f980bd31f1366c541b8c7490abaf55cd"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d02be208d2885ca085d4c852b911493b8cdac9d6eae893259da32bd72a437c25"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:655df442ceebfc6f3fd6c8766e04b60d44dddedfa90275d794f9fab2d3180879"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6e183729bb64be7798ccbfda6283ebf423c869268c25af2b56929e48f763be2f"},
|
||||
{file = "duckdb-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:61fb838da51e07ceb0222c4406b059b90e10efcc453c19a3650b73c0112138c4"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:7807e2f0d3344668e433f0dc1f54bfaddd410589611393e9a7ed56f8dec9514f"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:3da30b7b466f710d52caa1fdc3ef0bf4176ad7f115953cd9f8b0fbf0f723778f"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b9b6a77ef0183f561b1fc2945fcc762a71570ffd33fea4e3a855d413ed596fe4"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:16243e66a9fd0e64ee265f2634d137adc6593f54ddf3ef55cb8a29e1decf6e54"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42b910a149e00f40a1766dc74fa309d4255b912a5d2fdcc387287658048650f6"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47849d546dc4238c0f20e95fe53b621aa5b08684e68fff91fd84a7092be91a17"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11ec967b67159361ceade34095796a8d19368ea5c30cad988f44896b082b0816"},
|
||||
{file = "duckdb-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:510b5885ed6c267b9c0e1e7c6138fdffc2dd6f934a5a95b76da85da127213338"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:657bc7ac64d5faf069a782ae73afac51ef30ae2e5d0e09ce6a09d03db84ab35e"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:89f3de8cba57d19b41cd3c47dd06d979bd2a2ffead115480e37afbe72b02896d"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f6486323ab20656d22ffa8f3c6e109dde30d0b327b7c831f22ebcfe747f97fb0"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78a4510f82431ee3f14db689fe8727a4a9062c8f2fbb3bcfe3bfad3c1a198004"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64bf2a6e23840d662bd2ac09206a9bd4fa657418884d69e5c352d4456dc70b3c"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23fc9aa0af74e3803ed90c8d98280fd5bcac8c940592bf6288e8fd60fb051d00"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1f3aea31341ce400640dd522e4399b941f66df17e39884f446638fe958d6117c"},
|
||||
{file = "duckdb-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:3db4ab31c20de4edaef152930836b38e7662cd71370748fdf2c38ba9cf854dc4"},
|
||||
{file = "duckdb-1.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e3b6b4fe1edfe35f64f403a9f0ab75258cee35abd964356893ee37424174b7e4"},
|
||||
{file = "duckdb-1.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aad02f50d5a2020822d1638fc1a9bcf082056f11d2e15ccfc1c1ed4d0f85a3be"},
|
||||
{file = "duckdb-1.1.0-cp37-cp37m-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb66e9e7391801928ea134dcab12d2e4c97f2ce0391c603a3e480bbb15830bc8"},
|
||||
{file = "duckdb-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:069fb7bca459e31edb32a61f0eea95d7a8a766bef7b8318072563abf8e939593"},
|
||||
{file = "duckdb-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e39f9b7b62e64e10d421ff04480290a70129c38067d1a4f600e9212b10542c5a"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-macosx_12_0_arm64.whl", hash = "sha256:55ef98bcc7ba745752607f1b926e8d9b7ce32c42c423bbad10c44820aefe23a7"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-macosx_12_0_universal2.whl", hash = "sha256:e2a08175e43b865c1e9611efd18cacd29ddd69093de442b1ebdf312071df7719"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:0e3644b1f034012d82b9baa12a7ea306fe71dc6623731b28c753c4a617ff9499"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:211a33c1ddb5cc609f75eb43772b0b03b45d2fa89bec107e4715267ca907806a"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e74b6f8a5145abbf7e6c1a2a61f0adbcd493c19b358f524ec9a3cebdf362abb"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58f1633dd2c5af5088ae2d119418e200855d0699d84f2fae9d46d30f404bcead"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:d18caea926b1e301c29b140418fca697aad728129e269b4f82c2795a184549e1"},
|
||||
{file = "duckdb-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:cd9fb1408942411ad360f8414bc3fbf0091c396ca903d947a10f2e31324d5cbd"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:bd11bc899cebf5ff936d1276a2dfb7b7db08aba3bcc42924afeafc2163bddb43"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-macosx_12_0_universal2.whl", hash = "sha256:53825a63193c582a78c152ea53de8d145744ddbeea18f452625a82ebc33eb14a"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:29dc18087de47563b3859a6b98bbed96e1c96ce5db829646dc3b16a916997e7d"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecb19319883564237a7a03a104dbe7f445e73519bb67108fcab3d19b6b91fe30"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aac2fcabe2d5072c252d0b3087365f431de812d8199705089fb073e4d039d19c"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d89eaaa5df8a57e7d2bc1f4c46493bb1fee319a00155f2015810ad2ace6570ae"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d86a6926313913cd2cc7e08816d3e7f72ba340adf2959279b1a80058be6526d9"},
|
||||
{file = "duckdb-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:d8333f3e85fa2a0f1c222b752c2bd42ea875235ff88492f7bcbb6867d0f644eb"},
|
||||
{file = "duckdb-1.1.0.tar.gz", hash = "sha256:b4d4c12b1f98732151bd31377753e0da1a20f6423016d2d097d2e31953ec7c23"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "duckduckgo-search"
|
||||
version = "6.2.12"
|
||||
|
|
@ -2287,6 +2232,16 @@ django = ["dj-database-url", "dj-email-url", "django-cache-url"]
|
|||
lint = ["flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"]
|
||||
tests = ["dj-database-url", "dj-email-url", "django-cache-url", "pytest"]
|
||||
|
||||
[[package]]
|
||||
name = "events"
|
||||
version = "0.5"
|
||||
description = "Bringing the elegance of C# EventHandler to Python"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "Events-0.5-py3-none-any.whl", hash = "sha256:a7286af378ba3e46640ac9825156c93bdba7502174dd696090fdfcd4d80a1abd"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exceptiongroup"
|
||||
version = "1.2.2"
|
||||
|
|
@ -6499,6 +6454,30 @@ typing-extensions = ">=4.11,<5"
|
|||
[package.extras]
|
||||
datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
|
||||
|
||||
[[package]]
|
||||
name = "opensearch-py"
|
||||
version = "2.7.1"
|
||||
description = "Python client for OpenSearch"
|
||||
optional = false
|
||||
python-versions = "<4,>=3.8"
|
||||
files = [
|
||||
{file = "opensearch_py-2.7.1-py3-none-any.whl", hash = "sha256:5417650eba98a1c7648e502207cebf3a12beab623ffe0ebbf55f9b1b4b6e44e9"},
|
||||
{file = "opensearch_py-2.7.1.tar.gz", hash = "sha256:67ab76e9373669bc71da417096df59827c08369ac3795d5438c9a8be21cbd759"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
certifi = ">=2024.07.04"
|
||||
Events = "*"
|
||||
python-dateutil = "*"
|
||||
requests = ">=2.32.0,<3.0.0"
|
||||
urllib3 = {version = ">=1.26.19,<2.2.0 || >2.2.0,<2.2.1 || >2.2.1,<3", markers = "python_version >= \"3.10\""}
|
||||
|
||||
[package.extras]
|
||||
async = ["aiohttp (>=3.9.4,<4)"]
|
||||
develop = ["black (>=24.3.0)", "botocore", "coverage (<8.0.0)", "jinja2", "myst-parser", "pytest (>=3.0.0)", "pytest-cov", "pytest-mock (<4.0.0)", "pytz", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"]
|
||||
docs = ["aiohttp (>=3.9.4,<4)", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-rtd-theme"]
|
||||
kerberos = ["requests-kerberos"]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-api"
|
||||
version = "1.25.0"
|
||||
|
|
@ -12171,4 +12150,4 @@ local = ["ctransformers", "llama-cpp-python", "sentence-transformers"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = ">=3.10,<3.13"
|
||||
content-hash = "b8b616387b7386f9beab685fcff95ca507f802eb9dc22bd9ae182f0793b11170"
|
||||
content-hash = "4f9dbbdd9929d50f643790fbae6ecb8e2f052c2a0c399dd79efb6840450bf48d"
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ langchain-unstructured = "^0.1.2"
|
|||
pydantic-settings = "2.4.0"
|
||||
ragstack-ai-knowledge-store = "^0.2.1"
|
||||
duckduckgo-search = "^6.2.12"
|
||||
opensearch-py = "^2.7.1"
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
|
|
|
|||
268
src/backend/base/langflow/components/vectorstores/OpenSearch.py
Normal file
268
src/backend/base/langflow/components/vectorstores/OpenSearch.py
Normal file
|
|
@ -0,0 +1,268 @@
|
|||
import json
|
||||
import traceback
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from langchain_community.vectorstores import OpenSearchVectorSearch
|
||||
from loguru import logger
|
||||
|
||||
from langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
|
||||
from langflow.io import (
|
||||
BoolInput,
|
||||
DataInput,
|
||||
DropdownInput,
|
||||
FloatInput,
|
||||
HandleInput,
|
||||
IntInput,
|
||||
MultilineInput,
|
||||
SecretStrInput,
|
||||
StrInput,
|
||||
)
|
||||
from langflow.schema import Data
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain_community.vectorstores import OpenSearchVectorSearch
|
||||
|
||||
|
||||
class OpenSearchVectorStoreComponent(LCVectorStoreComponent):
|
||||
"""
|
||||
OpenSearch Vector Store with advanced, customizable search capabilities.
|
||||
"""
|
||||
|
||||
display_name: str = "OpenSearch"
|
||||
description: str = "OpenSearch Vector Store with advanced, customizable search capabilities."
|
||||
documentation = "https://python.langchain.com/docs/integrations/vectorstores/opensearch"
|
||||
name = "OpenSearch"
|
||||
icon = "OpenSearch"
|
||||
|
||||
inputs = [
|
||||
StrInput(
|
||||
name="opensearch_url",
|
||||
display_name="OpenSearch URL",
|
||||
value="http://localhost:9200",
|
||||
info="URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).",
|
||||
),
|
||||
StrInput(
|
||||
name="index_name",
|
||||
display_name="Index Name",
|
||||
value="langflow",
|
||||
info="The index name where the vectors will be stored in OpenSearch cluster.",
|
||||
),
|
||||
MultilineInput(
|
||||
name="search_input",
|
||||
display_name="Search Input",
|
||||
info=(
|
||||
"Enter a search query. Leave empty to retrieve all documents. "
|
||||
"If you need a more advanced search consider using Hybrid Search Query instead."
|
||||
),
|
||||
value="",
|
||||
),
|
||||
DataInput(
|
||||
name="ingest_data",
|
||||
display_name="Ingest Data",
|
||||
is_list=True,
|
||||
),
|
||||
HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
|
||||
DropdownInput(
|
||||
name="search_type",
|
||||
display_name="Search Type",
|
||||
options=["similarity", "similarity_score_threshold", "mmr"],
|
||||
value="similarity",
|
||||
advanced=True,
|
||||
),
|
||||
IntInput(
|
||||
name="number_of_results",
|
||||
display_name="Number of Results",
|
||||
info="Number of results to return.",
|
||||
advanced=True,
|
||||
value=4,
|
||||
),
|
||||
FloatInput(
|
||||
name="search_score_threshold",
|
||||
display_name="Search Score Threshold",
|
||||
info="Minimum similarity score threshold for search results.",
|
||||
value=0.0,
|
||||
advanced=True,
|
||||
),
|
||||
StrInput(
|
||||
name="username",
|
||||
display_name="Username",
|
||||
value="admin",
|
||||
advanced=True,
|
||||
),
|
||||
SecretStrInput(
|
||||
name="password",
|
||||
display_name="Password",
|
||||
value="admin",
|
||||
advanced=True,
|
||||
),
|
||||
BoolInput(
|
||||
name="use_ssl",
|
||||
display_name="Use SSL",
|
||||
value=True,
|
||||
advanced=True,
|
||||
),
|
||||
BoolInput(
|
||||
name="verify_certs",
|
||||
display_name="Verify Certificates",
|
||||
value=False,
|
||||
advanced=True,
|
||||
),
|
||||
MultilineInput(
|
||||
name="hybrid_search_query",
|
||||
display_name="Hybrid Search Query",
|
||||
value="",
|
||||
advanced=True,
|
||||
info=(
|
||||
"Provide a custom hybrid search query in JSON format. This allows you to combine "
|
||||
"vector similarity and keyword matching."
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
@check_cached_vector_store
|
||||
def build_vector_store(self) -> OpenSearchVectorSearch:
|
||||
"""
|
||||
Builds the OpenSearch Vector Store object.
|
||||
"""
|
||||
try:
|
||||
from langchain_community.vectorstores import OpenSearchVectorSearch
|
||||
except ImportError as e:
|
||||
error_message = f"Failed to import required modules: {str(e)}"
|
||||
logger.error(error_message)
|
||||
raise ImportError(error_message) from e
|
||||
|
||||
try:
|
||||
opensearch = OpenSearchVectorSearch(
|
||||
index_name=self.index_name,
|
||||
embedding_function=self.embedding,
|
||||
opensearch_url=self.opensearch_url,
|
||||
http_auth=(self.username, self.password),
|
||||
use_ssl=self.use_ssl,
|
||||
verify_certs=self.verify_certs,
|
||||
ssl_assert_hostname=False,
|
||||
ssl_show_warn=False,
|
||||
)
|
||||
except Exception as e:
|
||||
error_message = f"Failed to create OpenSearchVectorSearch instance: {str(e)}"
|
||||
logger.error(error_message)
|
||||
raise RuntimeError(error_message) from e
|
||||
|
||||
if self.ingest_data:
|
||||
self._add_documents_to_vector_store(opensearch)
|
||||
|
||||
return opensearch
|
||||
|
||||
def _add_documents_to_vector_store(self, vector_store: "OpenSearchVectorSearch") -> None:
|
||||
"""
|
||||
Adds documents to the Vector Store.
|
||||
"""
|
||||
documents = []
|
||||
for _input in self.ingest_data or []:
|
||||
if isinstance(_input, Data):
|
||||
documents.append(_input.to_lc_document())
|
||||
else:
|
||||
error_message = f"Expected Data object, got {type(_input)}"
|
||||
logger.error(error_message)
|
||||
raise ValueError(error_message)
|
||||
|
||||
if documents and self.embedding is not None:
|
||||
logger.debug(f"Adding {len(documents)} documents to the Vector Store.")
|
||||
try:
|
||||
vector_store.add_documents(documents)
|
||||
except Exception as e:
|
||||
error_message = f"Error adding documents to Vector Store: {str(e)}"
|
||||
logger.error(error_message)
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
raise RuntimeError(error_message) from e
|
||||
else:
|
||||
logger.debug("No documents to add to the Vector Store.")
|
||||
|
||||
def search(self, query: str | None = None) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Search for similar documents in the vector store or retrieve all documents if no query is provided.
|
||||
"""
|
||||
try:
|
||||
vector_store = self.build_vector_store()
|
||||
|
||||
query = query or ""
|
||||
|
||||
if self.hybrid_search_query.strip():
|
||||
try:
|
||||
hybrid_query = json.loads(self.hybrid_search_query)
|
||||
except json.JSONDecodeError as e:
|
||||
error_message = f"Invalid hybrid search query JSON: {str(e)}"
|
||||
logger.error(error_message)
|
||||
raise ValueError(error_message) from e
|
||||
|
||||
results = vector_store.client.search(index=self.index_name, body=hybrid_query)
|
||||
|
||||
processed_results = []
|
||||
for hit in results.get("hits", {}).get("hits", []):
|
||||
source = hit.get("_source", {})
|
||||
text = source.get("text", "")
|
||||
metadata = source.get("metadata", {})
|
||||
|
||||
if isinstance(text, dict):
|
||||
text = text.get("text", "")
|
||||
|
||||
processed_results.append(
|
||||
{
|
||||
"page_content": text,
|
||||
"metadata": metadata,
|
||||
}
|
||||
)
|
||||
return processed_results
|
||||
|
||||
search_kwargs = {"k": self.number_of_results}
|
||||
search_type = self.search_type.lower()
|
||||
|
||||
if search_type == "similarity":
|
||||
results = vector_store.similarity_search(query, **search_kwargs)
|
||||
return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
|
||||
if search_type == "similarity_score_threshold":
|
||||
search_kwargs["score_threshold"] = self.search_score_threshold
|
||||
results = vector_store.similarity_search_with_relevance_scores(query, **search_kwargs)
|
||||
return [
|
||||
{
|
||||
"page_content": doc.page_content,
|
||||
"metadata": doc.metadata,
|
||||
"score": score,
|
||||
}
|
||||
for doc, score in results
|
||||
]
|
||||
if search_type == "mmr":
|
||||
results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
|
||||
return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
|
||||
|
||||
error_message = f"Invalid search type:: {self.search_type}"
|
||||
logger.error(error_message)
|
||||
raise ValueError(error_message)
|
||||
|
||||
except Exception as e:
|
||||
error_message = f"Error during search: {str(e)}"
|
||||
logger.error(error_message)
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
raise RuntimeError(error_message) from e
|
||||
|
||||
def search_documents(self) -> list[Data]:
|
||||
"""
|
||||
Search for documents in the vector store based on the search input.
|
||||
If no search input is provided, retrieve all documents.
|
||||
"""
|
||||
try:
|
||||
query = self.search_input.strip() if self.search_input else None
|
||||
results = self.search(query)
|
||||
retrieved_data = [
|
||||
Data(
|
||||
file_path=result["metadata"].get("file_path", ""),
|
||||
text=result["page_content"],
|
||||
)
|
||||
for result in results
|
||||
]
|
||||
self.status = retrieved_data
|
||||
return retrieved_data
|
||||
except Exception as e:
|
||||
error_message = f"Error during document search: {str(e)}"
|
||||
logger.error(error_message)
|
||||
logger.error(f"Traceback: {traceback.format_exc()}")
|
||||
raise RuntimeError(error_message) from e
|
||||
149
src/frontend/src/icons/OpenSearch/OpenSearch.jsx
Normal file
149
src/frontend/src/icons/OpenSearch/OpenSearch.jsx
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
const OpenSearchSVG = (props) => (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
shapeRendering="geometricPrecision"
|
||||
textRendering="geometricPrecision"
|
||||
imageRendering="optimizeQuality"
|
||||
fillRule="evenodd"
|
||||
clipRule="evenodd"
|
||||
viewBox="0 0 384 384"
|
||||
width="1em"
|
||||
height="1em"
|
||||
{...props}
|
||||
>
|
||||
<defs>
|
||||
<clipPath id="clip-corners">
|
||||
<path
|
||||
d="
|
||||
M20,0
|
||||
H364
|
||||
L384,20
|
||||
V364
|
||||
L364,384
|
||||
H20
|
||||
L0,364
|
||||
V20
|
||||
L20,0
|
||||
Z
|
||||
"
|
||||
/>
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clipPath="url(#clip-corners)">
|
||||
<path
|
||||
fill="none"
|
||||
d="
|
||||
M184,385
|
||||
C122.667,385 61.833,385 1,385
|
||||
C1,257 1,129 1,1
|
||||
C129,1 257,1 385,1
|
||||
C385,129 385,257 385,385
|
||||
C318.167,385 251.333,385 184,385
|
||||
M152.632,273
|
||||
C159.84,272.45 167.291,272.877 174.204,271.151
|
||||
C189.36,267.366 201.162,250.941 201.297,233.963
|
||||
C201.419,218.668 195.233,208.643 181.681,203.039
|
||||
C172.325,199.17 162.661,195.941 152.941,193.083
|
||||
C134.986,187.803 116.803,183.11 101.915,170.935
|
||||
C90.816,161.858 81.722,151.386 81.018,136.047
|
||||
C80.801,131.315 80.984,126.564 80.984,121.823
|
||||
C80.546,121.619 80.107,121.415 79.669,121.212
|
||||
C79.154,122.032 78.595,122.827 78.13,123.675
|
||||
C61.223,154.499 59.805,186.125 74.625,217.906
|
||||
C83.52,236.981 97.714,251.737 116.262,262.043
|
||||
C127.296,268.175 138.983,272.214 152.632,273
|
||||
M220.971,157.484
|
||||
C226.154,160.754 231.74,163.533 236.44,167.391
|
||||
C247.615,176.563 256.672,187.243 257.04,202.803
|
||||
C257.141,207.093 257.055,211.388 257.055,215.682
|
||||
C259.515,214.005 260.824,212.155 261.907,210.181
|
||||
C272.642,190.62 275.244,169.652 272.001,147.894
|
||||
C268.042,121.329 254.573,100.205 233.584,83.838
|
||||
C216.339,70.391 196.84,63.823 174.716,64.887
|
||||
C161.575,65.518 151.592,71.048 143.835,81.336
|
||||
C137.299,90.003 136.524,99.967 136.983,110.283
|
||||
C137.253,116.343 139.324,121.551 143.478,125.998
|
||||
C152.822,136 165.641,138.909 177.89,142.993
|
||||
C192.029,147.706 206.17,152.415 220.971,157.484
|
||||
M269.535,282.983
|
||||
C274.183,278.328 279.456,274.138 283.35,268.92
|
||||
C291.277,258.299 299.752,247.774 305.668,236.035
|
||||
C316.346,214.846 322.14,192.102 321.025,168.012
|
||||
C320.776,162.62 318.447,159.497 313.848,159.012
|
||||
C307.257,158.317 304.314,160.117 302.958,166.033
|
||||
C302.377,168.565 302.32,171.215 301.997,173.809
|
||||
C301.046,181.456 300.442,189.169 299.043,196.734
|
||||
C296.359,211.245 290.559,224.615 282.944,237.204
|
||||
C270.562,257.673 253.973,273.887 233.027,285.413
|
||||
C213.571,296.119 192.819,302.354 170.367,302.053
|
||||
C168.28,302.025 166.114,302.509 164.105,303.138
|
||||
C160.282,304.336 158.141,309.059 159.061,314.285
|
||||
C159.772,318.321 162.303,321.135 166.452,320.898
|
||||
C177.456,320.269 188.552,319.897 199.393,318.104
|
||||
C215.51,315.438 230.725,309.57 244.62,300.952
|
||||
C253.103,295.691 260.909,289.339 269.535,282.983
|
||||
"
|
||||
/>
|
||||
<path
|
||||
fill="#005EB8"
|
||||
d="
|
||||
M152.153,273
|
||||
C138.983,272.214 127.296,268.175 116.262,262.043
|
||||
C97.714,251.737 83.52,236.981 74.625,217.906
|
||||
C59.805,186.125 61.223,154.499 78.13,123.675
|
||||
C78.595,122.827 79.154,122.032 79.669,121.212
|
||||
C80.107,121.415 80.546,121.619 80.984,121.823
|
||||
C80.984,126.564 80.801,131.315 81.018,136.047
|
||||
C81.722,151.386 90.816,161.858 101.915,170.935
|
||||
C116.803,183.11 134.986,187.803 152.941,193.083
|
||||
C162.661,195.941 172.325,199.17 181.681,203.039
|
||||
C195.233,208.643 201.419,218.668 201.297,233.963
|
||||
C201.162,250.941 189.36,267.366 174.204,271.151
|
||||
C167.291,272.877 159.84,272.45 152.153,273
|
||||
"
|
||||
/>
|
||||
<path
|
||||
fill="#003B5C"
|
||||
d="
|
||||
M220.64,157.305
|
||||
C206.17,152.415 192.029,147.706 177.89,142.993
|
||||
C165.641,138.909 152.822,136 143.478,125.998
|
||||
C139.324,121.551 137.253,116.343 136.983,110.283
|
||||
C136.524,99.967 137.299,90.003 143.835,81.336
|
||||
C151.592,71.048 161.575,65.518 174.716,64.887
|
||||
C196.84,63.823 216.339,70.391 233.584,83.838
|
||||
C254.573,100.205 268.042,121.329 272.001,147.894
|
||||
C275.244,169.652 272.642,190.62 261.907,210.181
|
||||
C260.824,212.155 259.515,214.005 257.055,215.682
|
||||
C257.055,211.388 257.141,207.093 257.04,202.803
|
||||
C256.672,187.243 247.615,176.563 236.44,167.391
|
||||
C231.74,163.533 226.154,160.754 220.64,157.305
|
||||
"
|
||||
/>
|
||||
<path
|
||||
fill="#005EB8"
|
||||
d="
|
||||
M269.277,283.23
|
||||
C260.909,289.339 253.103,295.691 244.62,300.952
|
||||
C230.725,309.57 215.51,315.438 199.393,318.104
|
||||
C188.552,319.897 177.456,320.269 166.452,320.898
|
||||
C162.303,321.135 159.772,318.321 159.061,314.285
|
||||
C158.141,309.059 160.282,304.336 164.105,303.138
|
||||
C166.114,302.509 168.28,302.025 170.367,302.053
|
||||
C192.819,302.354 213.571,296.119 233.027,285.413
|
||||
C253.973,273.887 270.562,257.673 282.944,237.204
|
||||
C290.559,224.615 296.359,211.245 299.043,196.734
|
||||
C300.442,189.169 301.046,181.456 301.997,173.809
|
||||
C302.32,171.215 302.377,168.565 302.958,166.033
|
||||
C304.314,160.117 307.257,158.317 313.848,159.012
|
||||
C318.447,159.497 320.776,162.62 321.025,168.012
|
||||
C322.14,192.102 316.346,214.846 305.668,236.035
|
||||
C299.752,247.774 291.277,258.299 283.35,268.92
|
||||
C279.456,274.138 274.183,278.328 269.277,283.23
|
||||
"
|
||||
/>
|
||||
</g>
|
||||
</svg>
|
||||
);
|
||||
|
||||
export default OpenSearchSVG;
|
||||
9
src/frontend/src/icons/OpenSearch/index.tsx
Normal file
9
src/frontend/src/icons/OpenSearch/index.tsx
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import React, { forwardRef } from "react";
|
||||
import OpenSearchSVG from "./OpenSearch";
|
||||
|
||||
export const OpenSearch = forwardRef<
|
||||
SVGSVGElement,
|
||||
React.PropsWithChildren<{}>
|
||||
>((props, ref) => {
|
||||
return <OpenSearchSVG ref={ref} {...props} />;
|
||||
});
|
||||
126
src/frontend/src/icons/OpenSearch/opensearch.svg
Normal file
126
src/frontend/src/icons/OpenSearch/opensearch.svg
Normal file
|
|
@ -0,0 +1,126 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg"
|
||||
shape-rendering="geometricPrecision"
|
||||
text-rendering="geometricPrecision"
|
||||
image-rendering="optimizeQuality"
|
||||
fill-rule="evenodd"
|
||||
clip-rule="evenodd"
|
||||
viewBox="0 0 384 384">
|
||||
<defs>
|
||||
<clipPath id="clip-corners">
|
||||
<path d="
|
||||
M20,0
|
||||
H364
|
||||
L384,20
|
||||
V364
|
||||
L364,384
|
||||
H20
|
||||
L0,364
|
||||
V20
|
||||
L20,0
|
||||
Z
|
||||
" />
|
||||
</clipPath>
|
||||
</defs>
|
||||
<g clip-path="url(#clip-corners)">
|
||||
<path fill="none" d="
|
||||
M184,385
|
||||
C122.667,385 61.833,385 1,385
|
||||
C1,257 1,129 1,1
|
||||
C129,1 257,1 385,1
|
||||
C385,129 385,257 385,385
|
||||
C318.167,385 251.333,385 184,385
|
||||
M152.632,273
|
||||
C159.84,272.45 167.291,272.877 174.204,271.151
|
||||
C189.36,267.366 201.162,250.941 201.297,233.963
|
||||
C201.419,218.668 195.233,208.643 181.681,203.039
|
||||
C172.325,199.17 162.661,195.941 152.941,193.083
|
||||
C134.986,187.803 116.803,183.11 101.915,170.935
|
||||
C90.816,161.858 81.722,151.386 81.018,136.047
|
||||
C80.801,131.315 80.984,126.564 80.984,121.823
|
||||
C80.546,121.619 80.107,121.415 79.669,121.212
|
||||
C79.154,122.032 78.595,122.827 78.13,123.675
|
||||
C61.223,154.499 59.805,186.125 74.625,217.906
|
||||
C83.52,236.981 97.714,251.737 116.262,262.043
|
||||
C127.296,268.175 138.983,272.214 152.632,273
|
||||
M220.971,157.484
|
||||
C226.154,160.754 231.74,163.533 236.44,167.391
|
||||
C247.615,176.563 256.672,187.243 257.04,202.803
|
||||
C257.141,207.093 257.055,211.388 257.055,215.682
|
||||
C259.515,214.005 260.824,212.155 261.907,210.181
|
||||
C272.642,190.62 275.244,169.652 272.001,147.894
|
||||
C268.042,121.329 254.573,100.205 233.584,83.838
|
||||
C216.339,70.391 196.84,63.823 174.716,64.887
|
||||
C161.575,65.518 151.592,71.048 143.835,81.336
|
||||
C137.299,90.003 136.524,99.967 136.983,110.283
|
||||
C137.253,116.343 139.324,121.551 143.478,125.998
|
||||
C152.822,136 165.641,138.909 177.89,142.993
|
||||
C192.029,147.706 206.17,152.415 220.971,157.484
|
||||
M269.535,282.983
|
||||
C274.183,278.328 279.456,274.138 283.35,268.92
|
||||
C291.277,258.299 299.752,247.774 305.668,236.035
|
||||
C316.346,214.846 322.14,192.102 321.025,168.012
|
||||
C320.776,162.62 318.447,159.497 313.848,159.012
|
||||
C307.257,158.317 304.314,160.117 302.958,166.033
|
||||
C302.377,168.565 302.32,171.215 301.997,173.809
|
||||
C301.046,181.456 300.442,189.169 299.043,196.734
|
||||
C296.359,211.245 290.559,224.615 282.944,237.204
|
||||
C270.562,257.673 253.973,273.887 233.027,285.413
|
||||
C213.571,296.119 192.819,302.354 170.367,302.053
|
||||
C168.28,302.025 166.114,302.509 164.105,303.138
|
||||
C160.282,304.336 158.141,309.059 159.061,314.285
|
||||
C159.772,318.321 162.303,321.135 166.452,320.898
|
||||
C177.456,320.269 188.552,319.897 199.393,318.104
|
||||
C215.51,315.438 230.725,309.57 244.62,300.952
|
||||
C253.103,295.691 260.909,289.339 269.535,282.983
|
||||
"/>
|
||||
<path fill="#005EB8" d="
|
||||
M152.153,273
|
||||
C138.983,272.214 127.296,268.175 116.262,262.043
|
||||
C97.714,251.737 83.52,236.981 74.625,217.906
|
||||
C59.805,186.125 61.223,154.499 78.13,123.675
|
||||
C78.595,122.827 79.154,122.032 79.669,121.212
|
||||
C80.107,121.415 80.546,121.619 80.984,121.823
|
||||
C80.984,126.564 80.801,131.315 81.018,136.047
|
||||
C81.722,151.386 90.816,161.858 101.915,170.935
|
||||
C116.803,183.11 134.986,187.803 152.941,193.083
|
||||
C162.661,195.941 172.325,199.17 181.681,203.039
|
||||
C195.233,208.643 201.419,218.668 201.297,233.963
|
||||
C201.162,250.941 189.36,267.366 174.204,271.151
|
||||
C167.291,272.877 159.84,272.45 152.153,273
|
||||
"/>
|
||||
<path fill="#003B5C" d="
|
||||
M220.64,157.305
|
||||
C206.17,152.415 192.029,147.706 177.89,142.993
|
||||
C165.641,138.909 152.822,136 143.478,125.998
|
||||
C139.324,121.551 137.253,116.343 136.983,110.283
|
||||
C136.524,99.967 137.299,90.003 143.835,81.336
|
||||
C151.592,71.048 161.575,65.518 174.716,64.887
|
||||
C196.84,63.823 216.339,70.391 233.584,83.838
|
||||
C254.573,100.205 268.042,121.329 272.001,147.894
|
||||
C275.244,169.652 272.642,190.62 261.907,210.181
|
||||
C260.824,212.155 259.515,214.005 257.055,215.682
|
||||
C257.055,211.388 257.141,207.093 257.04,202.803
|
||||
C256.672,187.243 247.615,176.563 236.44,167.391
|
||||
C231.74,163.533 226.154,160.754 220.64,157.305
|
||||
"/>
|
||||
<path fill="#005EB8" d="
|
||||
M269.277,283.23
|
||||
C260.909,289.339 253.103,295.691 244.62,300.952
|
||||
C230.725,309.57 215.51,315.438 199.393,318.104
|
||||
C188.552,319.897 177.456,320.269 166.452,320.898
|
||||
C162.303,321.135 159.772,318.321 159.061,314.285
|
||||
C158.141,309.059 160.282,304.336 164.105,303.138
|
||||
C166.114,302.509 168.28,302.025 170.367,302.053
|
||||
C192.819,302.354 213.571,296.119 233.027,285.413
|
||||
C253.973,273.887 270.562,257.673 282.944,237.204
|
||||
C290.559,224.615 296.359,211.245 299.043,196.734
|
||||
C300.442,189.169 301.046,181.456 301.997,173.809
|
||||
C302.32,171.215 302.377,168.565 302.958,166.033
|
||||
C304.314,160.117 307.257,158.317 313.848,159.012
|
||||
C318.447,159.497 320.776,162.62 321.025,168.012
|
||||
C322.14,192.102 316.346,214.846 305.668,236.035
|
||||
C299.752,247.774 291.277,258.299 283.35,268.92
|
||||
C279.456,274.138 274.183,278.328 269.277,283.23
|
||||
"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.1 KiB |
|
|
@ -214,6 +214,7 @@ import { NotionIcon } from "../icons/Notion";
|
|||
import { NvidiaIcon } from "../icons/Nvidia";
|
||||
import { OllamaIcon } from "../icons/Ollama";
|
||||
import { OpenAiIcon } from "../icons/OpenAi";
|
||||
import { OpenSearch } from "../icons/OpenSearch";
|
||||
import { PineconeIcon } from "../icons/Pinecone";
|
||||
import { PostgresIcon } from "../icons/Postgres";
|
||||
import { PythonIcon } from "../icons/Python";
|
||||
|
|
@ -631,4 +632,5 @@ export const nodeIconsLucide: iconsType = {
|
|||
Option: OptionIcon,
|
||||
Perplexity,
|
||||
DuckDuckGo: DuckDuckGoIcon,
|
||||
OpenSearch,
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue