From ae274a3e6df242d5488f27a645a9e09dcb9cb997 Mon Sep 17 00:00:00 2001 From: Edwin Jose Date: Mon, 25 Aug 2025 14:12:49 -0400 Subject: [PATCH] ref: Refactor vectorstore components structure (#9486) * Refactor vectorstore components structure Moved vectorstore components for Chroma, ClickHouse, Couchbase, DataStax, Elastic, Milvus, MongoDB, Pinecone, Qdrant, Supabase, Upstash, Vectara, and Weaviate into dedicated subfolders with __init__.py files for each. Updated Redis vectorstore implementation to reside in redis.py and removed the old vectorstores/redis.py. Adjusted starter project JSONs and frontend constants to reflect new module paths and sidebar entries for these vectorstores. * Refactor vectorstore components and add lazy imports Moved Datastax-related files from vectorstores to a dedicated datastax directory. Added lazy import logic to __init__.py files for chroma, clickhouse, couchbase, elastic, milvus, mongodb, pinecone, qdrant, supabase, upstash, vectara, and weaviate components. Cleaned up vectorstores/__init__.py to only include local and faiss components, improving modularity and import efficiency. * [autofix.ci] apply automated fixes * Refactor vectorstore components structure Moved FAISS, Cassandra, and pgvector components to dedicated subdirectories with lazy-loading __init__.py files. Updated imports and references throughout the backend and frontend to reflect new locations. Removed obsolete datastax Cassandra component. Added new sidebar bundle entries for FAISS, Cassandra, and pgvector in frontend constants and style utilities. * Add lazy imports and Redis chat memory component Refactored the Redis module to support lazy imports for RedisIndexChatMemory and RedisVectorStoreComponent, improving import efficiency. Added a new redis_chat.py file implementing RedisIndexChatMemory for chat message storage and retrieval using Redis. * Fix vector store astra imports * Revert package lock changes * More test fixes * Update test_vector_store_rag.py * Update test_dynamic_imports.py * Update vector_store_rag.py * Update test_dynamic_imports.py * Refactor the cassandra chat component * Fix frontend tests for bundle * Mark Local DB as legacy * Update inputComponent.spec.ts * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Eric Hare Co-authored-by: Carlos Coelho <80289056+carlosrcoelho@users.noreply.github.com> --- .../langflow/components/FAISS/__init__.py | 34 ++++++ .../{vectorstores => FAISS}/faiss.py | 0 .../langflow/components/cassandra/__init__.py | 40 +++++++ .../{vectorstores => cassandra}/cassandra.py | 0 .../cassandra_chat.py} | 0 .../cassandra_graph.py | 0 .../langflow/components/chroma/__init__.py | 34 ++++++ .../{vectorstores => chroma}/chroma.py | 0 .../components/clickhouse/__init__.py | 34 ++++++ .../clickhouse.py | 0 .../langflow/components/couchbase/__init__.py | 34 ++++++ .../{vectorstores => couchbase}/couchbase.py | 0 .../langflow/components/datastax/__init__.py | 6 +- .../{vectorstores => datastax}/astradb.py | 0 .../astradb_graph.py | 0 .../{vectorstores => datastax}/graph_rag.py | 0 .../{vectorstores => datastax}/hcd.py | 0 .../langflow/components/elastic/__init__.py | 37 +++++++ .../elasticsearch.py | 0 .../{vectorstores => elastic}/opensearch.py | 0 .../langflow/components/milvus/__init__.py | 34 ++++++ .../{vectorstores => milvus}/milvus.py | 0 .../langflow/components/mongodb/__init__.py | 34 ++++++ .../mongodb_atlas.py | 0 .../langflow/components/pgvector/__init__.py | 34 ++++++ .../{vectorstores => pgvector}/pgvector.py | 0 .../langflow/components/pinecone/__init__.py | 34 ++++++ .../{vectorstores => pinecone}/pinecone.py | 0 .../langflow/components/qdrant/__init__.py | 34 ++++++ .../{vectorstores => qdrant}/qdrant.py | 0 .../langflow/components/redis/__init__.py | 38 ++++++- .../base/langflow/components/redis/redis.py | 104 +++++++++++++----- .../langflow/components/redis/redis_chat.py | 43 ++++++++ .../langflow/components/supabase/__init__.py | 37 +++++++ .../{vectorstores => supabase}/supabase.py | 0 .../langflow/components/upstash/__init__.py | 34 ++++++ .../{vectorstores => upstash}/upstash.py | 0 .../langflow/components/vectara/__init__.py | 37 +++++++ .../{vectorstores => vectara}/vectara.py | 0 .../{vectorstores => vectara}/vectara_rag.py | 0 .../components/vectorstores/__init__.py | 69 ------------ .../components/vectorstores/local_db.py | 1 + .../langflow/components/vectorstores/redis.py | 89 --------------- .../langflow/components/weaviate/__init__.py | 34 ++++++ .../{vectorstores => weaviate}/weaviate.py | 0 .../starter_projects/Hybrid Search RAG.json | 2 +- .../starter_projects/Nvidia Remix.json | 2 +- .../starter_projects/Vector Store RAG.json | 4 +- .../starter_projects/vector_store_rag.py | 2 +- .../components/astra/test_astra_component.py | 2 +- .../test_dynamic_import_integration.py | 20 ++-- .../base/tools/test_vector_store_decorator.py | 2 +- .../components/test_all_modules_importable.py | 2 +- .../unit/components/test_dynamic_imports.py | 8 +- .../test_chroma_vector_store_component.py | 2 +- .../vectorstores/test_graph_rag_component.py | 2 +- .../vectorstores/test_mongodb_atlas.py | 2 +- .../starter_projects/test_vector_store_rag.py | 2 +- src/frontend/src/constants/constants.ts | 15 +++ src/frontend/src/utils/styleUtils.ts | 14 +++ .../tests/core/features/filterSidebar.spec.ts | 4 +- .../tests/core/unit/inputComponent.spec.ts | 4 +- .../features/filterEdge-shard-1.spec.ts | 3 +- 63 files changed, 743 insertions(+), 224 deletions(-) create mode 100644 src/backend/base/langflow/components/FAISS/__init__.py rename src/backend/base/langflow/components/{vectorstores => FAISS}/faiss.py (100%) create mode 100644 src/backend/base/langflow/components/cassandra/__init__.py rename src/backend/base/langflow/components/{vectorstores => cassandra}/cassandra.py (100%) rename src/backend/base/langflow/components/{datastax/cassandra.py => cassandra/cassandra_chat.py} (100%) rename src/backend/base/langflow/components/{vectorstores => cassandra}/cassandra_graph.py (100%) create mode 100644 src/backend/base/langflow/components/chroma/__init__.py rename src/backend/base/langflow/components/{vectorstores => chroma}/chroma.py (100%) create mode 100644 src/backend/base/langflow/components/clickhouse/__init__.py rename src/backend/base/langflow/components/{vectorstores => clickhouse}/clickhouse.py (100%) create mode 100644 src/backend/base/langflow/components/couchbase/__init__.py rename src/backend/base/langflow/components/{vectorstores => couchbase}/couchbase.py (100%) rename src/backend/base/langflow/components/{vectorstores => datastax}/astradb.py (100%) rename src/backend/base/langflow/components/{vectorstores => datastax}/astradb_graph.py (100%) rename src/backend/base/langflow/components/{vectorstores => datastax}/graph_rag.py (100%) rename src/backend/base/langflow/components/{vectorstores => datastax}/hcd.py (100%) create mode 100644 src/backend/base/langflow/components/elastic/__init__.py rename src/backend/base/langflow/components/{vectorstores => elastic}/elasticsearch.py (100%) rename src/backend/base/langflow/components/{vectorstores => elastic}/opensearch.py (100%) create mode 100644 src/backend/base/langflow/components/milvus/__init__.py rename src/backend/base/langflow/components/{vectorstores => milvus}/milvus.py (100%) create mode 100644 src/backend/base/langflow/components/mongodb/__init__.py rename src/backend/base/langflow/components/{vectorstores => mongodb}/mongodb_atlas.py (100%) create mode 100644 src/backend/base/langflow/components/pgvector/__init__.py rename src/backend/base/langflow/components/{vectorstores => pgvector}/pgvector.py (100%) create mode 100644 src/backend/base/langflow/components/pinecone/__init__.py rename src/backend/base/langflow/components/{vectorstores => pinecone}/pinecone.py (100%) create mode 100644 src/backend/base/langflow/components/qdrant/__init__.py rename src/backend/base/langflow/components/{vectorstores => qdrant}/qdrant.py (100%) create mode 100644 src/backend/base/langflow/components/redis/redis_chat.py create mode 100644 src/backend/base/langflow/components/supabase/__init__.py rename src/backend/base/langflow/components/{vectorstores => supabase}/supabase.py (100%) create mode 100644 src/backend/base/langflow/components/upstash/__init__.py rename src/backend/base/langflow/components/{vectorstores => upstash}/upstash.py (100%) create mode 100644 src/backend/base/langflow/components/vectara/__init__.py rename src/backend/base/langflow/components/{vectorstores => vectara}/vectara.py (100%) rename src/backend/base/langflow/components/{vectorstores => vectara}/vectara_rag.py (100%) delete mode 100644 src/backend/base/langflow/components/vectorstores/redis.py create mode 100644 src/backend/base/langflow/components/weaviate/__init__.py rename src/backend/base/langflow/components/{vectorstores => weaviate}/weaviate.py (100%) diff --git a/src/backend/base/langflow/components/FAISS/__init__.py b/src/backend/base/langflow/components/FAISS/__init__.py new file mode 100644 index 000000000..cb7246bbd --- /dev/null +++ b/src/backend/base/langflow/components/FAISS/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .faiss import FaissVectorStoreComponent + +_dynamic_imports = { + "FaissVectorStoreComponent": "faiss", +} + +__all__ = [ + "FaissVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import FAISS components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/faiss.py b/src/backend/base/langflow/components/FAISS/faiss.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/faiss.py rename to src/backend/base/langflow/components/FAISS/faiss.py diff --git a/src/backend/base/langflow/components/cassandra/__init__.py b/src/backend/base/langflow/components/cassandra/__init__.py new file mode 100644 index 000000000..5a07a6d13 --- /dev/null +++ b/src/backend/base/langflow/components/cassandra/__init__.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .cassandra import CassandraVectorStoreComponent + from .cassandra_chat import CassandraChatMemory + from .cassandra_graph import CassandraGraphVectorStoreComponent + +_dynamic_imports = { + "CassandraVectorStoreComponent": "cassandra", + "CassandraGraphVectorStoreComponent": "cassandra_graph", + "CassandraChatMemory": "cassandra_chat", +} + +__all__ = [ + "CassandraChatMemory", + "CassandraGraphVectorStoreComponent", + "CassandraVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Cassandra components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/cassandra.py b/src/backend/base/langflow/components/cassandra/cassandra.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/cassandra.py rename to src/backend/base/langflow/components/cassandra/cassandra.py diff --git a/src/backend/base/langflow/components/datastax/cassandra.py b/src/backend/base/langflow/components/cassandra/cassandra_chat.py similarity index 100% rename from src/backend/base/langflow/components/datastax/cassandra.py rename to src/backend/base/langflow/components/cassandra/cassandra_chat.py diff --git a/src/backend/base/langflow/components/vectorstores/cassandra_graph.py b/src/backend/base/langflow/components/cassandra/cassandra_graph.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/cassandra_graph.py rename to src/backend/base/langflow/components/cassandra/cassandra_graph.py diff --git a/src/backend/base/langflow/components/chroma/__init__.py b/src/backend/base/langflow/components/chroma/__init__.py new file mode 100644 index 000000000..2bd5f2324 --- /dev/null +++ b/src/backend/base/langflow/components/chroma/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .chroma import ChromaVectorStoreComponent + +_dynamic_imports = { + "ChromaVectorStoreComponent": "chroma", +} + +__all__ = [ + "ChromaVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Chroma components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/chroma.py b/src/backend/base/langflow/components/chroma/chroma.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/chroma.py rename to src/backend/base/langflow/components/chroma/chroma.py diff --git a/src/backend/base/langflow/components/clickhouse/__init__.py b/src/backend/base/langflow/components/clickhouse/__init__.py new file mode 100644 index 000000000..2245a7d39 --- /dev/null +++ b/src/backend/base/langflow/components/clickhouse/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .clickhouse import ClickhouseVectorStoreComponent + +_dynamic_imports = { + "ClickhouseVectorStoreComponent": "clickhouse", +} + +__all__ = [ + "ClickhouseVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import ClickHouse components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/clickhouse.py b/src/backend/base/langflow/components/clickhouse/clickhouse.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/clickhouse.py rename to src/backend/base/langflow/components/clickhouse/clickhouse.py diff --git a/src/backend/base/langflow/components/couchbase/__init__.py b/src/backend/base/langflow/components/couchbase/__init__.py new file mode 100644 index 000000000..890caddeb --- /dev/null +++ b/src/backend/base/langflow/components/couchbase/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .couchbase import CouchbaseVectorStoreComponent + +_dynamic_imports = { + "CouchbaseVectorStoreComponent": "couchbase", +} + +__all__ = [ + "CouchbaseVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Couchbase components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/couchbase.py b/src/backend/base/langflow/components/couchbase/couchbase.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/couchbase.py rename to src/backend/base/langflow/components/couchbase/couchbase.py diff --git a/src/backend/base/langflow/components/datastax/__init__.py b/src/backend/base/langflow/components/datastax/__init__.py index 7802311bb..30bf2951d 100644 --- a/src/backend/base/langflow/components/datastax/__init__.py +++ b/src/backend/base/langflow/components/datastax/__init__.py @@ -8,9 +8,9 @@ if TYPE_CHECKING: from .astra_assistant_manager import AstraAssistantManager from .astra_db import AstraDBChatMemory from .astra_vectorize import AstraVectorizeComponent + from .astradb import AstraDBVectorStoreComponent from .astradb_cql import AstraDBCQLToolComponent from .astradb_tool import AstraDBToolComponent - from .cassandra import CassandraChatMemory from .create_assistant import AssistantsCreateAssistant from .create_thread import AssistantsCreateThread from .dotenv import Dotenv @@ -29,8 +29,8 @@ _dynamic_imports = { "AstraDBCQLToolComponent": "astradb_cql", "AstraDBChatMemory": "astra_db", "AstraDBToolComponent": "astradb_tool", + "AstraDBVectorStoreComponent": "astradb", "AstraVectorizeComponent": "astra_vectorize", - "CassandraChatMemory": "cassandra", "Dotenv": "dotenv", "GetEnvVar": "getenvvar", } @@ -45,8 +45,8 @@ __all__ = [ "AstraDBCQLToolComponent", "AstraDBChatMemory", "AstraDBToolComponent", + "AstraDBVectorStoreComponent", "AstraVectorizeComponent", - "CassandraChatMemory", "Dotenv", "GetEnvVar", ] diff --git a/src/backend/base/langflow/components/vectorstores/astradb.py b/src/backend/base/langflow/components/datastax/astradb.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/astradb.py rename to src/backend/base/langflow/components/datastax/astradb.py diff --git a/src/backend/base/langflow/components/vectorstores/astradb_graph.py b/src/backend/base/langflow/components/datastax/astradb_graph.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/astradb_graph.py rename to src/backend/base/langflow/components/datastax/astradb_graph.py diff --git a/src/backend/base/langflow/components/vectorstores/graph_rag.py b/src/backend/base/langflow/components/datastax/graph_rag.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/graph_rag.py rename to src/backend/base/langflow/components/datastax/graph_rag.py diff --git a/src/backend/base/langflow/components/vectorstores/hcd.py b/src/backend/base/langflow/components/datastax/hcd.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/hcd.py rename to src/backend/base/langflow/components/datastax/hcd.py diff --git a/src/backend/base/langflow/components/elastic/__init__.py b/src/backend/base/langflow/components/elastic/__init__.py new file mode 100644 index 000000000..b3a9ca8e0 --- /dev/null +++ b/src/backend/base/langflow/components/elastic/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .elasticsearch import ElasticsearchVectorStoreComponent + from .opensearch import OpenSearchVectorStoreComponent + +_dynamic_imports = { + "ElasticsearchVectorStoreComponent": "elasticsearch", + "OpenSearchVectorStoreComponent": "opensearch", +} + +__all__ = [ + "ElasticsearchVectorStoreComponent", + "OpenSearchVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Elastic components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/elasticsearch.py b/src/backend/base/langflow/components/elastic/elasticsearch.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/elasticsearch.py rename to src/backend/base/langflow/components/elastic/elasticsearch.py diff --git a/src/backend/base/langflow/components/vectorstores/opensearch.py b/src/backend/base/langflow/components/elastic/opensearch.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/opensearch.py rename to src/backend/base/langflow/components/elastic/opensearch.py diff --git a/src/backend/base/langflow/components/milvus/__init__.py b/src/backend/base/langflow/components/milvus/__init__.py new file mode 100644 index 000000000..9b5404d65 --- /dev/null +++ b/src/backend/base/langflow/components/milvus/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .milvus import MilvusVectorStoreComponent + +_dynamic_imports = { + "MilvusVectorStoreComponent": "milvus", +} + +__all__ = [ + "MilvusVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Milvus components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/milvus.py b/src/backend/base/langflow/components/milvus/milvus.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/milvus.py rename to src/backend/base/langflow/components/milvus/milvus.py diff --git a/src/backend/base/langflow/components/mongodb/__init__.py b/src/backend/base/langflow/components/mongodb/__init__.py new file mode 100644 index 000000000..ef55fe9b5 --- /dev/null +++ b/src/backend/base/langflow/components/mongodb/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .mongodb_atlas import MongoVectorStoreComponent + +_dynamic_imports = { + "MongoVectorStoreComponent": "mongodb_atlas", +} + +__all__ = [ + "MongoVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import MongoDB components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/mongodb_atlas.py b/src/backend/base/langflow/components/mongodb/mongodb_atlas.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/mongodb_atlas.py rename to src/backend/base/langflow/components/mongodb/mongodb_atlas.py diff --git a/src/backend/base/langflow/components/pgvector/__init__.py b/src/backend/base/langflow/components/pgvector/__init__.py new file mode 100644 index 000000000..92b638eea --- /dev/null +++ b/src/backend/base/langflow/components/pgvector/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .pgvector import PGVectorStoreComponent + +_dynamic_imports = { + "PGVectorStoreComponent": "pgvector", +} + +__all__ = [ + "PGVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import pgvector components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/pgvector.py b/src/backend/base/langflow/components/pgvector/pgvector.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/pgvector.py rename to src/backend/base/langflow/components/pgvector/pgvector.py diff --git a/src/backend/base/langflow/components/pinecone/__init__.py b/src/backend/base/langflow/components/pinecone/__init__.py new file mode 100644 index 000000000..283a70035 --- /dev/null +++ b/src/backend/base/langflow/components/pinecone/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .pinecone import PineconeVectorStoreComponent + +_dynamic_imports = { + "PineconeVectorStoreComponent": "pinecone", +} + +__all__ = [ + "PineconeVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Pinecone components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/pinecone.py b/src/backend/base/langflow/components/pinecone/pinecone.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/pinecone.py rename to src/backend/base/langflow/components/pinecone/pinecone.py diff --git a/src/backend/base/langflow/components/qdrant/__init__.py b/src/backend/base/langflow/components/qdrant/__init__.py new file mode 100644 index 000000000..2ca50834a --- /dev/null +++ b/src/backend/base/langflow/components/qdrant/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .qdrant import QdrantVectorStoreComponent + +_dynamic_imports = { + "QdrantVectorStoreComponent": "qdrant", +} + +__all__ = [ + "QdrantVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Qdrant components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/qdrant.py b/src/backend/base/langflow/components/qdrant/qdrant.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/qdrant.py rename to src/backend/base/langflow/components/qdrant/qdrant.py diff --git a/src/backend/base/langflow/components/redis/__init__.py b/src/backend/base/langflow/components/redis/__init__.py index b5148e15f..3b6838296 100644 --- a/src/backend/base/langflow/components/redis/__init__.py +++ b/src/backend/base/langflow/components/redis/__init__.py @@ -1,3 +1,37 @@ -from .redis import RedisIndexChatMemory +from __future__ import annotations -__all__ = ["RedisIndexChatMemory"] +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .redis import RedisVectorStoreComponent + from .redis_chat import RedisIndexChatMemory + +_dynamic_imports = { + "RedisVectorStoreComponent": "redis", + "RedisIndexChatMemory": "redis_chat", +} + +__all__ = [ + "RedisIndexChatMemory", + "RedisVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Redis components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/redis/redis.py b/src/backend/base/langflow/components/redis/redis.py index 95b47da03..c8c5b6ff8 100644 --- a/src/backend/base/langflow/components/redis/redis.py +++ b/src/backend/base/langflow/components/redis/redis.py @@ -1,43 +1,89 @@ -from urllib import parse +from pathlib import Path -from langchain_community.chat_message_histories.redis import RedisChatMessageHistory +from langchain.text_splitter import CharacterTextSplitter +from langchain_community.vectorstores.redis import Redis -from langflow.base.memory.model import LCChatMemoryComponent -from langflow.field_typing.constants import Memory -from langflow.inputs.inputs import IntInput, MessageTextInput, SecretStrInput, StrInput +from langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store +from langflow.helpers.data import docs_to_data +from langflow.io import HandleInput, IntInput, SecretStrInput, StrInput +from langflow.schema.data import Data -class RedisIndexChatMemory(LCChatMemoryComponent): - display_name = "Redis Chat Memory" - description = "Retrieves and store chat messages from Redis." - name = "RedisChatMemory" +class RedisVectorStoreComponent(LCVectorStoreComponent): + """A custom component for implementing a Vector Store using Redis.""" + + display_name: str = "Redis" + description: str = "Implementation of Vector Store using Redis" + name = "Redis" icon = "Redis" inputs = [ + SecretStrInput(name="redis_server_url", display_name="Redis Server Connection String", required=True), StrInput( - name="host", display_name="hostname", required=True, value="localhost", info="IP address or hostname." + name="redis_index_name", + display_name="Redis Index", ), - IntInput(name="port", display_name="port", required=True, value=6379, info="Redis Port Number."), - StrInput(name="database", display_name="database", required=True, value="0", info="Redis database."), - MessageTextInput( - name="username", display_name="Username", value="", info="The Redis user name.", advanced=True + StrInput(name="code", display_name="Code", advanced=True), + StrInput( + name="schema", + display_name="Schema", ), - SecretStrInput( - name="password", display_name="Password", value="", info="The password for username.", advanced=True - ), - StrInput(name="key_prefix", display_name="Key prefix", info="Key prefix.", advanced=True), - MessageTextInput( - name="session_id", display_name="Session ID", info="Session ID for the message.", advanced=True + *LCVectorStoreComponent.inputs, + IntInput( + name="number_of_results", + display_name="Number of Results", + info="Number of results to return.", + value=4, + advanced=True, ), + HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]), ] - def build_message_history(self) -> Memory: - kwargs = {} - password: str | None = self.password - if self.key_prefix: - kwargs["key_prefix"] = self.key_prefix - if password: - password = parse.quote_plus(password) + @check_cached_vector_store + def build_vector_store(self) -> Redis: + # Convert DataFrame to Data if needed using parent's method + self.ingest_data = self._prepare_ingest_data() - url = f"redis://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" - return RedisChatMessageHistory(session_id=self.session_id, url=url, **kwargs) + documents = [] + for _input in self.ingest_data or []: + if isinstance(_input, Data): + documents.append(_input.to_lc_document()) + else: + documents.append(_input) + Path("docuemnts.txt").write_text(str(documents), encoding="utf-8") + + if not documents: + if self.schema is None: + msg = "If no documents are provided, a schema must be provided." + raise ValueError(msg) + redis_vs = Redis.from_existing_index( + embedding=self.embedding, + index_name=self.redis_index_name, + schema=self.schema, + key_prefix=None, + redis_url=self.redis_server_url, + ) + else: + text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0) + docs = text_splitter.split_documents(documents) + redis_vs = Redis.from_documents( + documents=docs, + embedding=self.embedding, + redis_url=self.redis_server_url, + index_name=self.redis_index_name, + ) + return redis_vs + + def search_documents(self) -> list[Data]: + vector_store = self.build_vector_store() + + if self.search_query and isinstance(self.search_query, str) and self.search_query.strip(): + docs = vector_store.similarity_search( + query=self.search_query, + k=self.number_of_results, + ) + + data = docs_to_data(docs) + self.status = data + return data + return [] diff --git a/src/backend/base/langflow/components/redis/redis_chat.py b/src/backend/base/langflow/components/redis/redis_chat.py new file mode 100644 index 000000000..95b47da03 --- /dev/null +++ b/src/backend/base/langflow/components/redis/redis_chat.py @@ -0,0 +1,43 @@ +from urllib import parse + +from langchain_community.chat_message_histories.redis import RedisChatMessageHistory + +from langflow.base.memory.model import LCChatMemoryComponent +from langflow.field_typing.constants import Memory +from langflow.inputs.inputs import IntInput, MessageTextInput, SecretStrInput, StrInput + + +class RedisIndexChatMemory(LCChatMemoryComponent): + display_name = "Redis Chat Memory" + description = "Retrieves and store chat messages from Redis." + name = "RedisChatMemory" + icon = "Redis" + + inputs = [ + StrInput( + name="host", display_name="hostname", required=True, value="localhost", info="IP address or hostname." + ), + IntInput(name="port", display_name="port", required=True, value=6379, info="Redis Port Number."), + StrInput(name="database", display_name="database", required=True, value="0", info="Redis database."), + MessageTextInput( + name="username", display_name="Username", value="", info="The Redis user name.", advanced=True + ), + SecretStrInput( + name="password", display_name="Password", value="", info="The password for username.", advanced=True + ), + StrInput(name="key_prefix", display_name="Key prefix", info="Key prefix.", advanced=True), + MessageTextInput( + name="session_id", display_name="Session ID", info="Session ID for the message.", advanced=True + ), + ] + + def build_message_history(self) -> Memory: + kwargs = {} + password: str | None = self.password + if self.key_prefix: + kwargs["key_prefix"] = self.key_prefix + if password: + password = parse.quote_plus(password) + + url = f"redis://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + return RedisChatMessageHistory(session_id=self.session_id, url=url, **kwargs) diff --git a/src/backend/base/langflow/components/supabase/__init__.py b/src/backend/base/langflow/components/supabase/__init__.py new file mode 100644 index 000000000..bffe1e8f2 --- /dev/null +++ b/src/backend/base/langflow/components/supabase/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .supabase import SupabaseVectorStoreComponent + from .supabase_composio import SupabaseComposioComponent + +_dynamic_imports = { + "SupabaseVectorStoreComponent": "supabase", + "SupabaseComposioComponent": "supabase_composio", +} + +__all__ = [ + "SupabaseComposioComponent", + "SupabaseVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Supabase components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/supabase.py b/src/backend/base/langflow/components/supabase/supabase.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/supabase.py rename to src/backend/base/langflow/components/supabase/supabase.py diff --git a/src/backend/base/langflow/components/upstash/__init__.py b/src/backend/base/langflow/components/upstash/__init__.py new file mode 100644 index 000000000..7e6cec80d --- /dev/null +++ b/src/backend/base/langflow/components/upstash/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .upstash import UpstashVectorStoreComponent + +_dynamic_imports = { + "UpstashVectorStoreComponent": "upstash", +} + +__all__ = [ + "UpstashVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Upstash components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/upstash.py b/src/backend/base/langflow/components/upstash/upstash.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/upstash.py rename to src/backend/base/langflow/components/upstash/upstash.py diff --git a/src/backend/base/langflow/components/vectara/__init__.py b/src/backend/base/langflow/components/vectara/__init__.py new file mode 100644 index 000000000..81c78df38 --- /dev/null +++ b/src/backend/base/langflow/components/vectara/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .vectara import VectaraVectorStoreComponent + from .vectara_rag import VectaraRagComponent + +_dynamic_imports = { + "VectaraVectorStoreComponent": "vectara", + "VectaraRagComponent": "vectara_rag", +} + +__all__ = [ + "VectaraRagComponent", + "VectaraVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Vectara components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/vectara.py b/src/backend/base/langflow/components/vectara/vectara.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/vectara.py rename to src/backend/base/langflow/components/vectara/vectara.py diff --git a/src/backend/base/langflow/components/vectorstores/vectara_rag.py b/src/backend/base/langflow/components/vectara/vectara_rag.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/vectara_rag.py rename to src/backend/base/langflow/components/vectara/vectara_rag.py diff --git a/src/backend/base/langflow/components/vectorstores/__init__.py b/src/backend/base/langflow/components/vectorstores/__init__.py index 8a4115e65..104e90427 100644 --- a/src/backend/base/langflow/components/vectorstores/__init__.py +++ b/src/backend/base/langflow/components/vectorstores/__init__.py @@ -5,83 +5,14 @@ from typing import TYPE_CHECKING, Any from langflow.components._importing import import_mod if TYPE_CHECKING: - from .astradb import AstraDBVectorStoreComponent - from .astradb_graph import AstraDBGraphVectorStoreComponent - from .cassandra import CassandraVectorStoreComponent - from .cassandra_graph import CassandraGraphVectorStoreComponent - from .chroma import ChromaVectorStoreComponent - from .clickhouse import ClickhouseVectorStoreComponent - from .couchbase import CouchbaseVectorStoreComponent - from .elasticsearch import ElasticsearchVectorStoreComponent - from .faiss import FaissVectorStoreComponent - from .graph_rag import GraphRAGComponent - from .hcd import HCDVectorStoreComponent from .local_db import LocalDBComponent - from .milvus import MilvusVectorStoreComponent - from .mongodb_atlas import MongoVectorStoreComponent - from .opensearch import OpenSearchVectorStoreComponent - from .pgvector import PGVectorStoreComponent - from .pinecone import PineconeVectorStoreComponent - from .qdrant import QdrantVectorStoreComponent - from .redis import RedisVectorStoreComponent - from .supabase import SupabaseVectorStoreComponent - from .upstash import UpstashVectorStoreComponent - from .vectara import VectaraVectorStoreComponent - from .vectara_rag import VectaraRagComponent - from .weaviate import WeaviateVectorStoreComponent _dynamic_imports = { - "AstraDBVectorStoreComponent": "astradb", - "AstraDBGraphVectorStoreComponent": "astradb_graph", - "CassandraVectorStoreComponent": "cassandra", - "CassandraGraphVectorStoreComponent": "cassandra_graph", - "ChromaVectorStoreComponent": "chroma", - "ClickhouseVectorStoreComponent": "clickhouse", - "CouchbaseVectorStoreComponent": "couchbase", - "ElasticsearchVectorStoreComponent": "elasticsearch", - "FaissVectorStoreComponent": "faiss", - "GraphRAGComponent": "graph_rag", - "HCDVectorStoreComponent": "hcd", "LocalDBComponent": "local_db", - "MilvusVectorStoreComponent": "milvus", - "MongoVectorStoreComponent": "mongodb_atlas", - "OpenSearchVectorStoreComponent": "opensearch", - "PGVectorStoreComponent": "pgvector", - "PineconeVectorStoreComponent": "pinecone", - "QdrantVectorStoreComponent": "qdrant", - "RedisVectorStoreComponent": "redis", - "SupabaseVectorStoreComponent": "supabase", - "UpstashVectorStoreComponent": "upstash", - "VectaraVectorStoreComponent": "vectara", - "VectaraRagComponent": "vectara_rag", - "WeaviateVectorStoreComponent": "weaviate", } __all__ = [ - "AstraDBGraphVectorStoreComponent", - "AstraDBVectorStoreComponent", - "CassandraGraphVectorStoreComponent", - "CassandraVectorStoreComponent", - "ChromaVectorStoreComponent", - "ClickhouseVectorStoreComponent", - "CouchbaseVectorStoreComponent", - "ElasticsearchVectorStoreComponent", - "FaissVectorStoreComponent", - "GraphRAGComponent", - "HCDVectorStoreComponent", "LocalDBComponent", - "MilvusVectorStoreComponent", - "MongoVectorStoreComponent", - "OpenSearchVectorStoreComponent", - "PGVectorStoreComponent", - "PineconeVectorStoreComponent", - "QdrantVectorStoreComponent", - "RedisVectorStoreComponent", - "SupabaseVectorStoreComponent", - "UpstashVectorStoreComponent", - "VectaraRagComponent", - "VectaraVectorStoreComponent", - "WeaviateVectorStoreComponent", ] diff --git a/src/backend/base/langflow/components/vectorstores/local_db.py b/src/backend/base/langflow/components/vectorstores/local_db.py index f73e50323..7dedc2bed 100644 --- a/src/backend/base/langflow/components/vectorstores/local_db.py +++ b/src/backend/base/langflow/components/vectorstores/local_db.py @@ -21,6 +21,7 @@ class LocalDBComponent(LCVectorStoreComponent): description: str = "Local Vector Store with search capabilities" name = "LocalDB" icon = "database" + legacy = True inputs = [ TabInput( diff --git a/src/backend/base/langflow/components/vectorstores/redis.py b/src/backend/base/langflow/components/vectorstores/redis.py deleted file mode 100644 index c8c5b6ff8..000000000 --- a/src/backend/base/langflow/components/vectorstores/redis.py +++ /dev/null @@ -1,89 +0,0 @@ -from pathlib import Path - -from langchain.text_splitter import CharacterTextSplitter -from langchain_community.vectorstores.redis import Redis - -from langflow.base.vectorstores.model import LCVectorStoreComponent, check_cached_vector_store -from langflow.helpers.data import docs_to_data -from langflow.io import HandleInput, IntInput, SecretStrInput, StrInput -from langflow.schema.data import Data - - -class RedisVectorStoreComponent(LCVectorStoreComponent): - """A custom component for implementing a Vector Store using Redis.""" - - display_name: str = "Redis" - description: str = "Implementation of Vector Store using Redis" - name = "Redis" - icon = "Redis" - - inputs = [ - SecretStrInput(name="redis_server_url", display_name="Redis Server Connection String", required=True), - StrInput( - name="redis_index_name", - display_name="Redis Index", - ), - StrInput(name="code", display_name="Code", advanced=True), - StrInput( - name="schema", - display_name="Schema", - ), - *LCVectorStoreComponent.inputs, - IntInput( - name="number_of_results", - display_name="Number of Results", - info="Number of results to return.", - value=4, - advanced=True, - ), - HandleInput(name="embedding", display_name="Embedding", input_types=["Embeddings"]), - ] - - @check_cached_vector_store - def build_vector_store(self) -> Redis: - # Convert DataFrame to Data if needed using parent's method - self.ingest_data = self._prepare_ingest_data() - - documents = [] - for _input in self.ingest_data or []: - if isinstance(_input, Data): - documents.append(_input.to_lc_document()) - else: - documents.append(_input) - Path("docuemnts.txt").write_text(str(documents), encoding="utf-8") - - if not documents: - if self.schema is None: - msg = "If no documents are provided, a schema must be provided." - raise ValueError(msg) - redis_vs = Redis.from_existing_index( - embedding=self.embedding, - index_name=self.redis_index_name, - schema=self.schema, - key_prefix=None, - redis_url=self.redis_server_url, - ) - else: - text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0) - docs = text_splitter.split_documents(documents) - redis_vs = Redis.from_documents( - documents=docs, - embedding=self.embedding, - redis_url=self.redis_server_url, - index_name=self.redis_index_name, - ) - return redis_vs - - def search_documents(self) -> list[Data]: - vector_store = self.build_vector_store() - - if self.search_query and isinstance(self.search_query, str) and self.search_query.strip(): - docs = vector_store.similarity_search( - query=self.search_query, - k=self.number_of_results, - ) - - data = docs_to_data(docs) - self.status = data - return data - return [] diff --git a/src/backend/base/langflow/components/weaviate/__init__.py b/src/backend/base/langflow/components/weaviate/__init__.py new file mode 100644 index 000000000..440147be1 --- /dev/null +++ b/src/backend/base/langflow/components/weaviate/__init__.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from langflow.components._importing import import_mod + +if TYPE_CHECKING: + from .weaviate import WeaviateVectorStoreComponent + +_dynamic_imports = { + "WeaviateVectorStoreComponent": "weaviate", +} + +__all__ = [ + "WeaviateVectorStoreComponent", +] + + +def __getattr__(attr_name: str) -> Any: + """Lazily import Weaviate components on attribute access.""" + if attr_name not in _dynamic_imports: + msg = f"module '{__name__}' has no attribute '{attr_name}'" + raise AttributeError(msg) + try: + result = import_mod(attr_name, _dynamic_imports[attr_name], __spec__.parent) + except (ModuleNotFoundError, ImportError, AttributeError) as e: + msg = f"Could not import '{attr_name}' from '{__name__}': {e}" + raise AttributeError(msg) from e + globals()[attr_name] = result + return result + + +def __dir__() -> list[str]: + return list(__all__) diff --git a/src/backend/base/langflow/components/vectorstores/weaviate.py b/src/backend/base/langflow/components/weaviate/weaviate.py similarity index 100% rename from src/backend/base/langflow/components/vectorstores/weaviate.py rename to src/backend/base/langflow/components/weaviate/weaviate.py diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Hybrid Search RAG.json b/src/backend/base/langflow/initial_setup/starter_projects/Hybrid Search RAG.json index 6d3292a02..ed7435015 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Hybrid Search RAG.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Hybrid Search RAG.json @@ -1199,7 +1199,7 @@ "lf_version": "1.4.3", "metadata": { "code_hash": "23fbe9daca09", - "module": "langflow.components.vectorstores.astradb.AstraDBVectorStoreComponent" + "module": "langflow.components.datastax.astradb.AstraDBVectorStoreComponent" }, "minimized": false, "output_types": [], diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Nvidia Remix.json b/src/backend/base/langflow/initial_setup/starter_projects/Nvidia Remix.json index 2d4300dd1..ee3e7d1ec 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Nvidia Remix.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Nvidia Remix.json @@ -2183,7 +2183,7 @@ "lf_version": "1.4.2", "metadata": { "code_hash": "ed38680af3a6", - "module": "langflow.components.vectorstores.faiss.FaissVectorStoreComponent" + "module": "langflow.components.FAISS.faiss.FaissVectorStoreComponent" }, "minimized": false, "output_types": [], diff --git a/src/backend/base/langflow/initial_setup/starter_projects/Vector Store RAG.json b/src/backend/base/langflow/initial_setup/starter_projects/Vector Store RAG.json index a62534b38..45dddf829 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/Vector Store RAG.json +++ b/src/backend/base/langflow/initial_setup/starter_projects/Vector Store RAG.json @@ -2710,7 +2710,7 @@ "legacy": false, "metadata": { "code_hash": "23fbe9daca09", - "module": "langflow.components.vectorstores.astradb.AstraDBVectorStoreComponent" + "module": "langflow.components.datastax.astradb.AstraDBVectorStoreComponent" }, "minimized": false, "output_types": [], @@ -3486,7 +3486,7 @@ "legacy": false, "metadata": { "code_hash": "23fbe9daca09", - "module": "langflow.components.vectorstores.astradb.AstraDBVectorStoreComponent" + "module": "langflow.components.datastax.astradb.AstraDBVectorStoreComponent" }, "minimized": false, "output_types": [], diff --git a/src/backend/base/langflow/initial_setup/starter_projects/vector_store_rag.py b/src/backend/base/langflow/initial_setup/starter_projects/vector_store_rag.py index 14b018598..9d3f43add 100644 --- a/src/backend/base/langflow/initial_setup/starter_projects/vector_store_rag.py +++ b/src/backend/base/langflow/initial_setup/starter_projects/vector_store_rag.py @@ -1,12 +1,12 @@ from textwrap import dedent from langflow.components.data import FileComponent +from langflow.components.datastax import AstraDBVectorStoreComponent from langflow.components.input_output import ChatInput, ChatOutput from langflow.components.models import LanguageModelComponent from langflow.components.openai.openai import OpenAIEmbeddingsComponent from langflow.components.processing import ParserComponent, PromptComponent from langflow.components.processing.split_text import SplitTextComponent -from langflow.components.vectorstores import AstraDBVectorStoreComponent from langflow.graph import Graph diff --git a/src/backend/tests/integration/components/astra/test_astra_component.py b/src/backend/tests/integration/components/astra/test_astra_component.py index b8c7da3dc..40535d819 100644 --- a/src/backend/tests/integration/components/astra/test_astra_component.py +++ b/src/backend/tests/integration/components/astra/test_astra_component.py @@ -4,8 +4,8 @@ import pytest from astrapy import DataAPIClient from langchain_astradb import AstraDBVectorStore, VectorServiceOptions from langchain_core.documents import Document +from langflow.components.datastax import AstraDBVectorStoreComponent from langflow.components.openai.openai import OpenAIEmbeddingsComponent -from langflow.components.vectorstores import AstraDBVectorStoreComponent from langflow.schema.data import Data from tests.api_keys import get_astradb_api_endpoint, get_astradb_application_token, get_openai_api_key diff --git a/src/backend/tests/integration/test_dynamic_import_integration.py b/src/backend/tests/integration/test_dynamic_import_integration.py index b5c2e3317..6d12353f0 100644 --- a/src/backend/tests/integration/test_dynamic_import_integration.py +++ b/src/backend/tests/integration/test_dynamic_import_integration.py @@ -112,7 +112,7 @@ class TestDynamicImportIntegration: # Time the import of a large module start_time = time.time() - from langflow.components import vectorstores + from langflow.components import chroma import_time = time.time() - start_time @@ -124,7 +124,7 @@ class TestDynamicImportIntegration: # Now access a component - this should trigger loading start_time = time.time() - chroma_component = vectorstores.ChromaVectorStoreComponent + chroma_component = chroma.ChromaVectorStoreComponent access_time = time.time() - start_time assert chroma_component is not None @@ -236,21 +236,21 @@ class TestDynamicImportIntegration: def test_large_scale_component_access(self): """Test accessing many components doesn't cause issues.""" - from langflow.components import vectorstores + from langflow.components import datastax # Access multiple components rapidly components_accessed = [] component_names = [ - "ChromaVectorStoreComponent", - "PineconeVectorStoreComponent", - "FaissVectorStoreComponent", - "WeaviateVectorStoreComponent", - "QdrantVectorStoreComponent", + "AstraDBVectorStoreComponent", + "AstraDBChatComponent", + "AstraDBToolComponent", + "AstraDBCQLToolComponent", + "AstraAssistantManager", ] for name in component_names: - if hasattr(vectorstores, name): - component = getattr(vectorstores, name) + if hasattr(datastax, name): + component = getattr(datastax, name) components_accessed.append(component) # Should have accessed multiple components without issues diff --git a/src/backend/tests/unit/base/tools/test_vector_store_decorator.py b/src/backend/tests/unit/base/tools/test_vector_store_decorator.py index a99f408cd..9d3e9371b 100644 --- a/src/backend/tests/unit/base/tools/test_vector_store_decorator.py +++ b/src/backend/tests/unit/base/tools/test_vector_store_decorator.py @@ -1,7 +1,7 @@ from typing import Any import pytest -from langflow.components.vectorstores import AstraDBVectorStoreComponent +from langflow.components.datastax import AstraDBVectorStoreComponent from tests.base import ComponentTestBaseWithoutClient, VersionComponentMapping diff --git a/src/backend/tests/unit/components/test_all_modules_importable.py b/src/backend/tests/unit/components/test_all_modules_importable.py index 876e88498..a0d61b7b6 100644 --- a/src/backend/tests/unit/components/test_all_modules_importable.py +++ b/src/backend/tests/unit/components/test_all_modules_importable.py @@ -287,7 +287,7 @@ class TestSpecificModulePatterns: import time # Test large modules - large_modules = ["vectorstores", "processing", "langchain_utilities"] + large_modules = ["data", "processing", "langchain_utilities"] for module_name in large_modules: if module_name in components.__all__: diff --git a/src/backend/tests/unit/components/test_dynamic_imports.py b/src/backend/tests/unit/components/test_dynamic_imports.py index e51016c81..faf0a2db0 100644 --- a/src/backend/tests/unit/components/test_dynamic_imports.py +++ b/src/backend/tests/unit/components/test_dynamic_imports.py @@ -209,17 +209,17 @@ class TestPerformanceCharacteristics: def test_lazy_loading_performance(self): """Test that components can be accessed and cached properly.""" - from langflow.components import vectorstores + from langflow.components import chroma as chromamodules # Test that we can access a component - chroma = vectorstores.ChromaVectorStoreComponent + chroma = chromamodules.ChromaVectorStoreComponent assert chroma is not None # After access, it should be cached in the module's globals - assert "ChromaVectorStoreComponent" in vectorstores.__dict__ + assert "ChromaVectorStoreComponent" in chromamodules.__dict__ # Subsequent access should return the same cached object - chroma_2 = vectorstores.ChromaVectorStoreComponent + chroma_2 = chromamodules.ChromaVectorStoreComponent assert chroma_2 is chroma def test_caching_behavior(self): diff --git a/src/backend/tests/unit/components/vectorstores/test_chroma_vector_store_component.py b/src/backend/tests/unit/components/vectorstores/test_chroma_vector_store_component.py index ab45d6f2f..8e1bf3ce9 100644 --- a/src/backend/tests/unit/components/vectorstores/test_chroma_vector_store_component.py +++ b/src/backend/tests/unit/components/vectorstores/test_chroma_vector_store_component.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import Any import pytest -from langflow.components.vectorstores.chroma import ChromaVectorStoreComponent +from langflow.components.chroma import ChromaVectorStoreComponent from langflow.schema.data import Data from tests.base import ComponentTestBaseWithoutClient, VersionComponentMapping diff --git a/src/backend/tests/unit/components/vectorstores/test_graph_rag_component.py b/src/backend/tests/unit/components/vectorstores/test_graph_rag_component.py index 9bdf2c636..8d9cfc889 100644 --- a/src/backend/tests/unit/components/vectorstores/test_graph_rag_component.py +++ b/src/backend/tests/unit/components/vectorstores/test_graph_rag_component.py @@ -5,7 +5,7 @@ from faker import Faker from langchain_community.embeddings.fake import DeterministicFakeEmbedding from langchain_core.documents import Document from langchain_core.vectorstores.in_memory import InMemoryVectorStore -from langflow.components.vectorstores.graph_rag import GraphRAGComponent +from langflow.components.datastax.graph_rag import GraphRAGComponent from tests.base import ComponentTestBaseWithoutClient diff --git a/src/backend/tests/unit/components/vectorstores/test_mongodb_atlas.py b/src/backend/tests/unit/components/vectorstores/test_mongodb_atlas.py index 1a7f32108..c609a949f 100644 --- a/src/backend/tests/unit/components/vectorstores/test_mongodb_atlas.py +++ b/src/backend/tests/unit/components/vectorstores/test_mongodb_atlas.py @@ -4,7 +4,7 @@ from typing import Any import pytest from langchain_community.embeddings.fake import DeterministicFakeEmbedding -from langflow.components.vectorstores.mongodb_atlas import MongoVectorStoreComponent +from langflow.components.mongodb import MongoVectorStoreComponent from langflow.schema.data import Data from pymongo.collection import Collection diff --git a/src/backend/tests/unit/initial_setup/starter_projects/test_vector_store_rag.py b/src/backend/tests/unit/initial_setup/starter_projects/test_vector_store_rag.py index 9dcff1778..ba08d1d1d 100644 --- a/src/backend/tests/unit/initial_setup/starter_projects/test_vector_store_rag.py +++ b/src/backend/tests/unit/initial_setup/starter_projects/test_vector_store_rag.py @@ -4,12 +4,12 @@ from textwrap import dedent import pytest from langflow.components.data import FileComponent +from langflow.components.datastax import AstraDBVectorStoreComponent from langflow.components.input_output import ChatInput, ChatOutput from langflow.components.openai.openai import OpenAIEmbeddingsComponent from langflow.components.openai.openai_chat_model import OpenAIModelComponent from langflow.components.processing import ParseDataComponent, PromptComponent from langflow.components.processing.split_text import SplitTextComponent -from langflow.components.vectorstores import AstraDBVectorStoreComponent from langflow.graph.graph.base import Graph from langflow.graph.graph.constants import Finish from langflow.schema import Data diff --git a/src/frontend/src/constants/constants.ts b/src/frontend/src/constants/constants.ts index d91d2c7fa..a297e194d 100644 --- a/src/frontend/src/constants/constants.ts +++ b/src/frontend/src/constants/constants.ts @@ -763,6 +763,21 @@ export const BUNDLES_SIDEBAR_FOLDER_NAMES = [ "langwatch", "YouTube", "youtube", + "pinecone", + "weaviate", + "qdrant", + "mongodb", + "elastic", + "supabase", + "milvus", + "chroma", + "clickhouse", + "couchbase", + "upstash", + "vectara", + "cassandra", + "FAISS", + "pgvector", ]; export const AUTHORIZED_DUPLICATE_REQUESTS = [ diff --git a/src/frontend/src/utils/styleUtils.ts b/src/frontend/src/utils/styleUtils.ts index 26c312a92..02c80e5cf 100644 --- a/src/frontend/src/utils/styleUtils.ts +++ b/src/frontend/src/utils/styleUtils.ts @@ -247,17 +247,23 @@ export const SIDEBAR_BUNDLES = [ { display_name: "Azure", name: "azure", icon: "Azure" }, { display_name: "Baidu", name: "baidu", icon: "BaiduQianfan" }, { display_name: "Bing", name: "bing", icon: "Bing" }, + { display_name: "Cassandra", name: "cassandra", icon: "Cassandra" }, + { display_name: "Chroma", name: "chroma", icon: "Chroma" }, + { display_name: "ClickHouse", name: "clickhouse", icon: "Clickhouse" }, { display_name: "Cleanlab", name: "cleanlab", icon: "Cleanlab" }, { display_name: "Cloudflare", name: "cloudflare", icon: "Cloudflare" }, { display_name: "Cohere", name: "cohere", icon: "Cohere" }, { display_name: "Composio", name: "composio", icon: "Composio" }, { display_name: "Confluence", name: "confluence", icon: "Confluence" }, + { display_name: "Couchbase", name: "couchbase", icon: "Couchbase" }, { display_name: "CrewAI", name: "crewai", icon: "CrewAI" }, { display_name: "DataStax", name: "datastax", icon: "AstraDB" }, { display_name: "DeepSeek", name: "deepseek", icon: "DeepSeek" }, { display_name: "Docling", name: "docling", icon: "Docling" }, { display_name: "DuckDuckGo", name: "duckduckgo", icon: "DuckDuckGo" }, + { display_name: "Elastic", name: "elastic", icon: "ElasticsearchStore" }, { display_name: "Exa", name: "exa", icon: "Exa" }, + { display_name: "FAISS", name: "FAISS", icon: "FAISS" }, { display_name: "Firecrawl", name: "firecrawl", icon: "FirecrawlCrawlApi" }, { display_name: "Git", name: "git", icon: "GitLoader" }, { display_name: "Glean", name: "glean", icon: "Glean" }, @@ -279,7 +285,9 @@ export const SIDEBAR_BUNDLES = [ { display_name: "MariTalk", name: "maritalk", icon: "Maritalk" }, { display_name: "Mem0", name: "mem0", icon: "Mem0" }, { display_name: "Memories", name: "memories", icon: "Cpu" }, + { display_name: "Milvus", name: "milvus", icon: "Milvus" }, { display_name: "MistralAI", name: "mistral", icon: "MistralAI" }, + { display_name: "MongoDB", name: "mongodb", icon: "MongoDB" }, { display_name: "Needle", name: "needle", icon: "Needle" }, { display_name: "Not Diamond", name: "notdiamond", icon: "NotDiamond" }, { display_name: "Notion", name: "Notion", icon: "Notion" }, @@ -290,17 +298,23 @@ export const SIDEBAR_BUNDLES = [ { display_name: "OpenAI", name: "openai", icon: "OpenAI" }, { display_name: "OpenRouter", name: "openrouter", icon: "OpenRouter" }, { display_name: "Perplexity", name: "perplexity", icon: "Perplexity" }, + { display_name: "pgvector", name: "pgvector", icon: "cpu" }, + { display_name: "Pinecone", name: "pinecone", icon: "Pinecone" }, + { display_name: "Qdrant", name: "qdrant", icon: "Qdrant" }, { display_name: "Redis", name: "redis", icon: "Redis" }, { display_name: "SambaNova", name: "sambanova", icon: "SambaNova" }, { display_name: "ScrapeGraph AI", name: "scrapegraph", icon: "ScrapeGraph" }, { display_name: "SearchApi", name: "searchapi", icon: "SearchAPI" }, { display_name: "SerpApi", name: "serpapi", icon: "SerpSearch" }, { display_name: "Serper", name: "serper", icon: "Serper" }, + { display_name: "Supabase", name: "supabase", icon: "Supabase" }, { display_name: "Tavily", name: "tavily", icon: "TavilyIcon" }, { display_name: "TwelveLabs", name: "twelvelabs", icon: "TwelveLabs" }, { display_name: "Unstructured", name: "unstructured", icon: "Unstructured" }, + { display_name: "Upstash", name: "upstash", icon: "Upstash" }, { display_name: "Vectara", name: "vectara", icon: "Vectara" }, { display_name: "Vector Stores", name: "vectorstores", icon: "Layers" }, + { display_name: "Weaviate", name: "weaviate", icon: "Weaviate" }, { display_name: "Vertex AI", name: "vertexai", icon: "VertexAI" }, { display_name: "Wikipedia", name: "wikipedia", icon: "Wikipedia" }, { diff --git a/src/frontend/tests/core/features/filterSidebar.spec.ts b/src/frontend/tests/core/features/filterSidebar.spec.ts index 683337f59..1f3cdd604 100644 --- a/src/frontend/tests/core/features/filterSidebar.spec.ts +++ b/src/frontend/tests/core/features/filterSidebar.spec.ts @@ -121,7 +121,7 @@ test( await expect(page.getByTestId("disclosure-tools")).toBeVisible(); await expect(page.getByTestId("dataAPI Request")).toBeVisible(); - await expect(page.getByTestId("vectorstoresAstra DB")).toBeVisible(); + await expect(page.getByTestId("datastaxAstra DB")).toBeVisible(); await expect(page.getByTestId("logicSub Flow [Deprecated]")).toBeVisible(); await page.getByTestId("sidebar-options-trigger").click(); @@ -137,7 +137,7 @@ test( await page.getByTestId("icon-X").first().click(); await expect(page.getByTestId("dataAPI Request")).not.toBeVisible(); - await expect(page.getByTestId("vectorstoresAstra DB")).not.toBeVisible(); + await expect(page.getByTestId("datastaxAstra DB")).not.toBeVisible(); await expect( page.getByTestId("logicSub Flow [Deprecated]"), ).not.toBeVisible(); diff --git a/src/frontend/tests/core/unit/inputComponent.spec.ts b/src/frontend/tests/core/unit/inputComponent.spec.ts index fb3d985c0..772ec9071 100644 --- a/src/frontend/tests/core/unit/inputComponent.spec.ts +++ b/src/frontend/tests/core/unit/inputComponent.spec.ts @@ -15,11 +15,11 @@ test( await page.getByTestId("sidebar-search-input").click(); await page.getByTestId("sidebar-search-input").fill("Chroma"); - await page.waitForSelector('[data-testid="vectorstoresChroma DB"]', { + await page.waitForSelector('[data-testid="chromaChroma DB"]', { timeout: 3000, }); await page - .getByTestId("vectorstoresChroma DB") + .getByTestId("chromaChroma DB") .dragTo(page.locator('//*[@id="react-flow-id"]')); await page.mouse.up(); await page.mouse.down(); diff --git a/src/frontend/tests/extended/features/filterEdge-shard-1.spec.ts b/src/frontend/tests/extended/features/filterEdge-shard-1.spec.ts index 663096c94..a14571523 100644 --- a/src/frontend/tests/extended/features/filterEdge-shard-1.spec.ts +++ b/src/frontend/tests/extended/features/filterEdge-shard-1.spec.ts @@ -65,13 +65,12 @@ test( "disclosure-bundles-langchain", "disclosure-bundles-assemblyai", "disclosure-bundles-datastax", - "disclosure-bundles-vector stores", ]; const elementTestIds = [ "input_outputChat Output", "dataAPI Request", - "vectorstoresAstra DB Graph", + "datastaxAstra DB", "langchain_utilitiesTool Calling Agent", "langchain_utilitiesConversationChain", "mem0Mem0 Chat Memory",