jyapayne/langflow
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: Add OpenSearch VectorStore Component with Ingest and Search Capabilities (#3799)

Browse source 
* update: adding opensearch-py dependency

* feat: adding OpenSearch icon

* feat: adding OpenSearch VectorStore code

* fix: removing unused methods and adding hybrid search capabilities using search method

* update: poetry lock with recent hash added

* fix: removing unused methods and adding hybrid search capabilities using search method

* fix: default value of search_input to an empty string

* fix: adapting code to match make format script
This commit is contained in:
João 2024-10-01 18:42:55 -03:00 • committed by GitHub
commit b9bcb09e63
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 590 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

268
src/backend/base/langflow/components/vectorstores/OpenSearch.py Normal file
View file

@ -0,0 +1,268 @@
import json
import traceback
from typing import TYPE_CHECKING, Any
from langchain_community.vectorstores import OpenSearchVectorSearch
from loguru import logger
from langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store
from langflow.io import (
BoolInput,
DataInput,
DropdownInput,
FloatInput,
HandleInput,
IntInput,
MultilineInput,
SecretStrInput,
StrInput,
)
from langflow.schema import Data
if TYPE_CHECKING:
from langchain_community.vectorstores import OpenSearchVectorSearch
class OpenSearchVectorStoreComponent(LCVectorStoreComponent):
"""
OpenSearch Vector Store with advanced, customizable search capabilities.
"""
display_name: str = "OpenSearch"
description: str = "OpenSearch Vector Store with advanced, customizable search capabilities."
documentation = "https://python.langchain.com/docs/integrations/vectorstores/opensearch"
name = "OpenSearch"
icon = "OpenSearch"
inputs = [
StrInput(
name="opensearch_url",
display_name="OpenSearch URL",
value="http://localhost:9200",
info="URL for OpenSearch cluster (e.g. https://192.168.1.1:9200).",
),
StrInput(
name="index_name",
display_name="Index Name",
value="langflow",
info="The index name where the vectors will be stored in OpenSearch cluster.",
),
MultilineInput(
name="search_input",
display_name="Search Input",
info=(
"Enter a search query. Leave empty to retrieve all documents. "
"If you need a more advanced search consider using Hybrid Search Query instead."
),
value="",
),
DataInput(
name="ingest_data",
display_name="Ingest Data",
is_list=True,
),
HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]),
DropdownInput(
name="search_type",
display_name="Search Type",
options=["similarity", "similarity_score_threshold", "mmr"],
value="similarity",
advanced=True,
),
IntInput(
name="number_of_results",
display_name="Number of Results",
info="Number of results to return.",
advanced=True,
value=4,
),
FloatInput(
name="search_score_threshold",
display_name="Search Score Threshold",
info="Minimum similarity score threshold for search results.",
value=0.0,
advanced=True,
),
StrInput(
name="username",
display_name="Username",
value="admin",
advanced=True,
),
SecretStrInput(
name="password",
display_name="Password",
value="admin",
advanced=True,
),
BoolInput(
name="use_ssl",
display_name="Use SSL",
value=True,
advanced=True,
),
BoolInput(
name="verify_certs",
display_name="Verify Certificates",
value=False,
advanced=True,
),
MultilineInput(
name="hybrid_search_query",
display_name="Hybrid Search Query",
value="",
advanced=True,
info=(
"Provide a custom hybrid search query in JSON format. This allows you to combine "
"vector similarity and keyword matching."
),
),
]
@check_cached_vector_store
def build_vector_store(self) -> OpenSearchVectorSearch:
"""
Builds the OpenSearch Vector Store object.
"""
try:
from langchain_community.vectorstores import OpenSearchVectorSearch
except ImportError as e:
error_message = f"Failed to import required modules: {str(e)}"
logger.error(error_message)
raise ImportError(error_message) from e
try:
opensearch = OpenSearchVectorSearch(
index_name=self.index_name,
embedding_function=self.embedding,
opensearch_url=self.opensearch_url,
http_auth=(self.username, self.password),
use_ssl=self.use_ssl,
verify_certs=self.verify_certs,
ssl_assert_hostname=False,
ssl_show_warn=False,
)
except Exception as e:
error_message = f"Failed to create OpenSearchVectorSearch instance: {str(e)}"
logger.error(error_message)
raise RuntimeError(error_message) from e
if self.ingest_data:
self._add_documents_to_vector_store(opensearch)
return opensearch
def _add_documents_to_vector_store(self, vector_store: "OpenSearchVectorSearch") -> None:
"""
Adds documents to the Vector Store.
"""
documents = []
for _input in self.ingest_data or []:
if isinstance(_input, Data):
documents.append(_input.to_lc_document())
else:
error_message = f"Expected Data object, got {type(_input)}"
logger.error(error_message)
raise ValueError(error_message)
if documents and self.embedding is not None:
logger.debug(f"Adding {len(documents)} documents to the Vector Store.")
try:
vector_store.add_documents(documents)
except Exception as e:
error_message = f"Error adding documents to Vector Store: {str(e)}"
logger.error(error_message)
logger.error(f"Traceback: {traceback.format_exc()}")
raise RuntimeError(error_message) from e
else:
logger.debug("No documents to add to the Vector Store.")
def search(self, query: str | None = None) -> list[dict[str, Any]]:
"""
Search for similar documents in the vector store or retrieve all documents if no query is provided.
"""
try:
vector_store = self.build_vector_store()
query = query or ""
if self.hybrid_search_query.strip():
try:
hybrid_query = json.loads(self.hybrid_search_query)
except json.JSONDecodeError as e:
error_message = f"Invalid hybrid search query JSON: {str(e)}"
logger.error(error_message)
raise ValueError(error_message) from e
results = vector_store.client.search(index=self.index_name, body=hybrid_query)
processed_results = []
for hit in results.get("hits", {}).get("hits", []):
source = hit.get("_source", {})
text = source.get("text", "")
metadata = source.get("metadata", {})
if isinstance(text, dict):
text = text.get("text", "")
processed_results.append(
{
"page_content": text,
"metadata": metadata,
}
)
return processed_results
search_kwargs = {"k": self.number_of_results}
search_type = self.search_type.lower()
if search_type == "similarity":
results = vector_store.similarity_search(query, **search_kwargs)
return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
if search_type == "similarity_score_threshold":
search_kwargs["score_threshold"] = self.search_score_threshold
results = vector_store.similarity_search_with_relevance_scores(query, **search_kwargs)
return [
{
"page_content": doc.page_content,
"metadata": doc.metadata,
"score": score,
}
for doc, score in results
]
if search_type == "mmr":
results = vector_store.max_marginal_relevance_search(query, **search_kwargs)
return [{"page_content": doc.page_content, "metadata": doc.metadata} for doc in results]
error_message = f"Invalid search type:: {self.search_type}"
logger.error(error_message)
raise ValueError(error_message)
except Exception as e:
error_message = f"Error during search: {str(e)}"
logger.error(error_message)
logger.error(f"Traceback: {traceback.format_exc()}")
raise RuntimeError(error_message) from e
def search_documents(self) -> list[Data]:
"""
Search for documents in the vector store based on the search input.
If no search input is provided, retrieve all documents.
"""
try:
query = self.search_input.strip() if self.search_input else None
results = self.search(query)
retrieved_data = [
Data(
file_path=result["metadata"].get("file_path", ""),
text=result["page_content"],
)
for result in results
]
self.status = retrieved_data
return retrieved_data
except Exception as e:
error_message = f"Error during document search: {str(e)}"
logger.error(error_message)
logger.error(f"Traceback: {traceback.format_exc()}")
raise RuntimeError(error_message) from e

149
src/frontend/src/icons/OpenSearch/OpenSearch.jsx Normal file
View file

@ -0,0 +1,149 @@
const OpenSearchSVG = (props) => (
<svg
xmlns="http://www.w3.org/2000/svg"
shapeRendering="geometricPrecision"
textRendering="geometricPrecision"
imageRendering="optimizeQuality"
fillRule="evenodd"
clipRule="evenodd"
viewBox="0 0 384 384"
width="1em"
height="1em"
{...props}
>
<defs>
<clipPath id="clip-corners">
<path
d="
M20,0
H364
L384,20
V364
L364,384
H20
L0,364
V20
L20,0
Z
"
/>
</clipPath>
</defs>
<g clipPath="url(#clip-corners)">
<path
fill="none"
d="
M184,385
C122.667,385 61.833,385 1,385
C1,257 1,129 1,1
C129,1 257,1 385,1
C385,129 385,257 385,385
C318.167,385 251.333,385 184,385
M152.632,273
C159.84,272.45 167.291,272.877 174.204,271.151
C189.36,267.366 201.162,250.941 201.297,233.963
C201.419,218.668 195.233,208.643 181.681,203.039
C172.325,199.17 162.661,195.941 152.941,193.083
C134.986,187.803 116.803,183.11 101.915,170.935
C90.816,161.858 81.722,151.386 81.018,136.047
C80.801,131.315 80.984,126.564 80.984,121.823
C80.546,121.619 80.107,121.415 79.669,121.212
C79.154,122.032 78.595,122.827 78.13,123.675
C61.223,154.499 59.805,186.125 74.625,217.906
C83.52,236.981 97.714,251.737 116.262,262.043
C127.296,268.175 138.983,272.214 152.632,273
M220.971,157.484
C226.154,160.754 231.74,163.533 236.44,167.391
C247.615,176.563 256.672,187.243 257.04,202.803
C257.141,207.093 257.055,211.388 257.055,215.682
C259.515,214.005 260.824,212.155 261.907,210.181
C272.642,190.62 275.244,169.652 272.001,147.894
C268.042,121.329 254.573,100.205 233.584,83.838
C216.339,70.391 196.84,63.823 174.716,64.887
C161.575,65.518 151.592,71.048 143.835,81.336
C137.299,90.003 136.524,99.967 136.983,110.283
C137.253,116.343 139.324,121.551 143.478,125.998
C152.822,136 165.641,138.909 177.89,142.993
C192.029,147.706 206.17,152.415 220.971,157.484
M269.535,282.983
C274.183,278.328 279.456,274.138 283.35,268.92
C291.277,258.299 299.752,247.774 305.668,236.035
C316.346,214.846 322.14,192.102 321.025,168.012
C320.776,162.62 318.447,159.497 313.848,159.012
C307.257,158.317 304.314,160.117 302.958,166.033
C302.377,168.565 302.32,171.215 301.997,173.809
C301.046,181.456 300.442,189.169 299.043,196.734
C296.359,211.245 290.559,224.615 282.944,237.204
C270.562,257.673 253.973,273.887 233.027,285.413
C213.571,296.119 192.819,302.354 170.367,302.053
C168.28,302.025 166.114,302.509 164.105,303.138
C160.282,304.336 158.141,309.059 159.061,314.285
C159.772,318.321 162.303,321.135 166.452,320.898
C177.456,320.269 188.552,319.897 199.393,318.104
C215.51,315.438 230.725,309.57 244.62,300.952
C253.103,295.691 260.909,289.339 269.535,282.983
"
/>
<path
fill="#005EB8"
d="
M152.153,273
C138.983,272.214 127.296,268.175 116.262,262.043
C97.714,251.737 83.52,236.981 74.625,217.906
C59.805,186.125 61.223,154.499 78.13,123.675
C78.595,122.827 79.154,122.032 79.669,121.212
C80.107,121.415 80.546,121.619 80.984,121.823
C80.984,126.564 80.801,131.315 81.018,136.047
C81.722,151.386 90.816,161.858 101.915,170.935
C116.803,183.11 134.986,187.803 152.941,193.083
C162.661,195.941 172.325,199.17 181.681,203.039
C195.233,208.643 201.419,218.668 201.297,233.963
C201.162,250.941 189.36,267.366 174.204,271.151
C167.291,272.877 159.84,272.45 152.153,273
"
/>
<path
fill="#003B5C"
d="
M220.64,157.305
C206.17,152.415 192.029,147.706 177.89,142.993
C165.641,138.909 152.822,136 143.478,125.998
C139.324,121.551 137.253,116.343 136.983,110.283
C136.524,99.967 137.299,90.003 143.835,81.336
C151.592,71.048 161.575,65.518 174.716,64.887
C196.84,63.823 216.339,70.391 233.584,83.838
C254.573,100.205 268.042,121.329 272.001,147.894
C275.244,169.652 272.642,190.62 261.907,210.181
C260.824,212.155 259.515,214.005 257.055,215.682
C257.055,211.388 257.141,207.093 257.04,202.803
C256.672,187.243 247.615,176.563 236.44,167.391
C231.74,163.533 226.154,160.754 220.64,157.305
"
/>
<path
fill="#005EB8"
d="
M269.277,283.23
C260.909,289.339 253.103,295.691 244.62,300.952
C230.725,309.57 215.51,315.438 199.393,318.104
C188.552,319.897 177.456,320.269 166.452,320.898
C162.303,321.135 159.772,318.321 159.061,314.285
C158.141,309.059 160.282,304.336 164.105,303.138
C166.114,302.509 168.28,302.025 170.367,302.053
C192.819,302.354 213.571,296.119 233.027,285.413
C253.973,273.887 270.562,257.673 282.944,237.204
C290.559,224.615 296.359,211.245 299.043,196.734
C300.442,189.169 301.046,181.456 301.997,173.809
C302.32,171.215 302.377,168.565 302.958,166.033
C304.314,160.117 307.257,158.317 313.848,159.012
C318.447,159.497 320.776,162.62 321.025,168.012
C322.14,192.102 316.346,214.846 305.668,236.035
C299.752,247.774 291.277,258.299 283.35,268.92
C279.456,274.138 274.183,278.328 269.277,283.23
"
/>
</g>
</svg>
);
export default OpenSearchSVG;

9
src/frontend/src/icons/OpenSearch/index.tsx Normal file
View file

@ -0,0 +1,9 @@
import React, { forwardRef } from "react";
import OpenSearchSVG from "./OpenSearch";
export const OpenSearch = forwardRef<
SVGSVGElement,
React.PropsWithChildren<{}>
>((props, ref) => {
return <OpenSearchSVG ref={ref} {...props} />;
});

126
src/frontend/src/icons/OpenSearch/opensearch.svg Normal file
View file

@ -0,0 +1,126 @@
<svg xmlns="http://www.w3.org/2000/svg"
shape-rendering="geometricPrecision"
text-rendering="geometricPrecision"
image-rendering="optimizeQuality"
fill-rule="evenodd"
clip-rule="evenodd"
viewBox="0 0 384 384">
<defs>
<clipPath id="clip-corners">
<path d="
M20,0
H364
L384,20
V364
L364,384
H20
L0,364
V20
L20,0
Z
" />
</clipPath>
</defs>
<g clip-path="url(#clip-corners)">
<path fill="none" d="
M184,385
C122.667,385 61.833,385 1,385
C1,257 1,129 1,1
C129,1 257,1 385,1
C385,129 385,257 385,385
C318.167,385 251.333,385 184,385
M152.632,273
C159.84,272.45 167.291,272.877 174.204,271.151
C189.36,267.366 201.162,250.941 201.297,233.963
C201.419,218.668 195.233,208.643 181.681,203.039
C172.325,199.17 162.661,195.941 152.941,193.083
C134.986,187.803 116.803,183.11 101.915,170.935
C90.816,161.858 81.722,151.386 81.018,136.047
C80.801,131.315 80.984,126.564 80.984,121.823
C80.546,121.619 80.107,121.415 79.669,121.212
C79.154,122.032 78.595,122.827 78.13,123.675
C61.223,154.499 59.805,186.125 74.625,217.906
C83.52,236.981 97.714,251.737 116.262,262.043
C127.296,268.175 138.983,272.214 152.632,273
M220.971,157.484
C226.154,160.754 231.74,163.533 236.44,167.391
C247.615,176.563 256.672,187.243 257.04,202.803
C257.141,207.093 257.055,211.388 257.055,215.682
C259.515,214.005 260.824,212.155 261.907,210.181
C272.642,190.62 275.244,169.652 272.001,147.894
C268.042,121.329 254.573,100.205 233.584,83.838
C216.339,70.391 196.84,63.823 174.716,64.887
C161.575,65.518 151.592,71.048 143.835,81.336
C137.299,90.003 136.524,99.967 136.983,110.283
C137.253,116.343 139.324,121.551 143.478,125.998
C152.822,136 165.641,138.909 177.89,142.993
C192.029,147.706 206.17,152.415 220.971,157.484
M269.535,282.983
C274.183,278.328 279.456,274.138 283.35,268.92
C291.277,258.299 299.752,247.774 305.668,236.035
C316.346,214.846 322.14,192.102 321.025,168.012
C320.776,162.62 318.447,159.497 313.848,159.012
C307.257,158.317 304.314,160.117 302.958,166.033
C302.377,168.565 302.32,171.215 301.997,173.809
C301.046,181.456 300.442,189.169 299.043,196.734
C296.359,211.245 290.559,224.615 282.944,237.204
C270.562,257.673 253.973,273.887 233.027,285.413
C213.571,296.119 192.819,302.354 170.367,302.053
C168.28,302.025 166.114,302.509 164.105,303.138
C160.282,304.336 158.141,309.059 159.061,314.285
C159.772,318.321 162.303,321.135 166.452,320.898
C177.456,320.269 188.552,319.897 199.393,318.104
C215.51,315.438 230.725,309.57 244.62,300.952
C253.103,295.691 260.909,289.339 269.535,282.983
"/>
<path fill="#005EB8" d="
M152.153,273
C138.983,272.214 127.296,268.175 116.262,262.043
C97.714,251.737 83.52,236.981 74.625,217.906
C59.805,186.125 61.223,154.499 78.13,123.675
C78.595,122.827 79.154,122.032 79.669,121.212
C80.107,121.415 80.546,121.619 80.984,121.823
C80.984,126.564 80.801,131.315 81.018,136.047
C81.722,151.386 90.816,161.858 101.915,170.935
C116.803,183.11 134.986,187.803 152.941,193.083
C162.661,195.941 172.325,199.17 181.681,203.039
C195.233,208.643 201.419,218.668 201.297,233.963
C201.162,250.941 189.36,267.366 174.204,271.151
C167.291,272.877 159.84,272.45 152.153,273
"/>
<path fill="#003B5C" d="
M220.64,157.305
C206.17,152.415 192.029,147.706 177.89,142.993
C165.641,138.909 152.822,136 143.478,125.998
C139.324,121.551 137.253,116.343 136.983,110.283
C136.524,99.967 137.299,90.003 143.835,81.336
C151.592,71.048 161.575,65.518 174.716,64.887
C196.84,63.823 216.339,70.391 233.584,83.838
C254.573,100.205 268.042,121.329 272.001,147.894
C275.244,169.652 272.642,190.62 261.907,210.181
C260.824,212.155 259.515,214.005 257.055,215.682
C257.055,211.388 257.141,207.093 257.04,202.803
C256.672,187.243 247.615,176.563 236.44,167.391
C231.74,163.533 226.154,160.754 220.64,157.305
"/>
<path fill="#005EB8" d="
M269.277,283.23
C260.909,289.339 253.103,295.691 244.62,300.952
C230.725,309.57 215.51,315.438 199.393,318.104
C188.552,319.897 177.456,320.269 166.452,320.898
C162.303,321.135 159.772,318.321 159.061,314.285
C158.141,309.059 160.282,304.336 164.105,303.138
C166.114,302.509 168.28,302.025 170.367,302.053
C192.819,302.354 213.571,296.119 233.027,285.413
C253.973,273.887 270.562,257.673 282.944,237.204
C290.559,224.615 296.359,211.245 299.043,196.734
C300.442,189.169 301.046,181.456 301.997,173.809
C302.32,171.215 302.377,168.565 302.958,166.033
C304.314,160.117 307.257,158.317 313.848,159.012
C318.447,159.497 320.776,162.62 321.025,168.012
C322.14,192.102 316.346,214.846 305.668,236.035
C299.752,247.774 291.277,258.299 283.35,268.92
C279.456,274.138 274.183,278.328 269.277,283.23
"/>
</g>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 5.1 KiB

2
src/frontend/src/utils/styleUtils.ts
View file

@ -214,6 +214,7 @@ import { NotionIcon } from "../icons/Notion";
import { NvidiaIcon } from "../icons/Nvidia";
import { OllamaIcon } from "../icons/Ollama";
import { OpenAiIcon } from "../icons/OpenAi";
import { OpenSearch } from "../icons/OpenSearch";
import { PineconeIcon } from "../icons/Pinecone";
import { PostgresIcon } from "../icons/Postgres";
import { PythonIcon } from "../icons/Python";
@ -631,4 +632,5 @@ export const nodeIconsLucide: iconsType = {
Option: OptionIcon,
Perplexity,
DuckDuckGo: DuckDuckGoIcon,
OpenSearch,
};