diff --git a/poetry.lock b/poetry.lock index 16c3652a2..4cdd7e898 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1030,6 +1030,27 @@ files = [ [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "dnspython" +version = "2.3.0" +description = "DNS toolkit" +category = "main" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {file = "dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] + +[package.extras] +curio = ["curio (>=1.2,<2.0)", "sniffio (>=1.1,<2.0)"] +dnssec = ["cryptography (>=2.6,<40.0)"] +doh = ["h2 (>=4.1.0)", "httpx (>=0.21.1)", "requests (>=2.23.0,<3.0.0)", "requests-toolbelt (>=0.9.1,<0.11.0)"] +doq = ["aioquic (>=0.9.20)"] +idna = ["idna (>=2.1,<4.0)"] +trio = ["trio (>=0.14,<0.23)"] +wmi = ["wmi (>=1.5.1,<2.0.0)"] + [[package]] name = "docarray" version = "0.21.0" @@ -2485,6 +2506,25 @@ files = [ [package.dependencies] typing-extensions = ">=4.5.0,<5.0.0" +[[package]] +name = "loguru" +version = "0.7.0" +description = "Python logging made (stupidly) simple" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.0-py3-none-any.whl", hash = "sha256:b93aa30099fa6860d4727f1b81f8718e965bb96253fa190fab2077aaad6d15d3"}, + {file = "loguru-0.7.0.tar.gz", hash = "sha256:1612053ced6ae84d7959dd7d5e431a0532642237ec21f7fd83ac73fe539e03e1"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (==5.3.0)", "colorama (==0.4.5)", "colorama (==0.4.6)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v0.990)", "pre-commit (==3.2.1)", "pytest (==6.1.2)", "pytest (==7.2.1)", "pytest-cov (==2.12.1)", "pytest-cov (==4.0.0)", "pytest-mypy-plugins (==1.10.1)", "pytest-mypy-plugins (==1.9.3)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.2.0)", "tox (==3.27.1)", "tox (==4.4.6)"] + [[package]] name = "lxml" version = "4.9.2" @@ -3806,6 +3846,32 @@ files = [ docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] +[[package]] +name = "pinecone-client" +version = "2.2.2" +description = "Pinecone client and SDK" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pinecone-client-2.2.2.tar.gz", hash = "sha256:391fe413754efd4e0ef00154b44271d63c4cdd4bedf088d23111a5725d863210"}, + {file = "pinecone_client-2.2.2-py3-none-any.whl", hash = "sha256:21fddb752668efee4d3c6b706346d9580e36a8b06b8d97afd60bd33ef2536e7e"}, +] + +[package.dependencies] +dnspython = ">=2.0.0" +loguru = ">=0.5.0" +numpy = ">=1.22.0" +python-dateutil = ">=2.5.3" +pyyaml = ">=5.4" +requests = ">=2.19.0" +tqdm = ">=4.64.1" +typing-extensions = ">=3.7.4" +urllib3 = ">=1.21.1" + +[package.extras] +grpc = ["googleapis-common-protos (>=1.53.0)", "grpc-gateway-protoc-gen-openapiv2 (==0.1.0)", "grpcio (>=1.44.0)", "lz4 (>=3.1.3)", "protobuf (>=3.19.5,<3.20.0)"] + [[package]] name = "platformdirs" version = "3.7.0" @@ -6100,6 +6166,21 @@ files = [ beautifulsoup4 = "*" requests = ">=2.0.0,<3.0.0" +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + [[package]] name = "wrapt" version = "1.15.0" @@ -6366,4 +6447,4 @@ deploy = ["langchain-serve"] [metadata] lock-version = "2.0" python-versions = ">=3.9,<3.12" -content-hash = "49c4252d510cda8418eb432e527180cb4b3581cbe8e67905150a6f86840704af" +content-hash = "51c72102b2106e55cdc2fce423897318a652985fba9a9e945cb9ec06397397a9" diff --git a/pyproject.toml b/pyproject.toml index f9159ae08..19d5a8f66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ multiprocess = "^0.70.14" cachetools = "^5.3.1" types-cachetools = "^5.3.0.5" appdirs = "^1.4.4" +pinecone-client = "^2.2.2" [tool.poetry.group.dev.dependencies] diff --git a/src/backend/langflow/interface/initialize/vector_store.py b/src/backend/langflow/interface/initialize/vector_store.py index c1ce5c227..dfdf3a28f 100644 --- a/src/backend/langflow/interface/initialize/vector_store.py +++ b/src/backend/langflow/interface/initialize/vector_store.py @@ -1,5 +1,6 @@ +import json from typing import Type -from langchain.vectorstores import Pinecone, Qdrant, Chroma +from langchain.vectorstores import Pinecone, Qdrant, Chroma, FAISS, Weaviate def docs_in_params(params: dict) -> bool: @@ -10,6 +11,45 @@ def docs_in_params(params: dict) -> bool: ) +def initialize_weaviate(class_object: Type[Weaviate], params: dict): + """Initialize weaviate and return the class object""" + if not docs_in_params(params): + import weaviate + + client_kwargs_json = params.get("client_kwargs", "{}") + client_kwargs = json.loads(client_kwargs_json) + client_params = { + "url": params.get("weaviate_url"), + } + client_params.update(client_kwargs) + weaviate_client = weaviate.Client(**client_params) + + new_params = { + "client": weaviate_client, + "index_name": params.get("index_name"), + "text_key": params.get("text_key"), + } + weaviate = class_object(**new_params) + # If there are docs in the params, create a new index + if "texts" in params: + params["documents"] = params.pop("texts") + + return class_object.from_documents(**params) + + +def initialize_faiss(class_object: Type[FAISS], params: dict): + """Initialize faiss and return the class object""" + + if not docs_in_params(params): + return class_object.load_local + + save_local = params.get("save_local") + faiss_index = class_object(**params) + if save_local: + faiss_index.save_local(folder_path=save_local) + return faiss_index + + def initialize_pinecone(class_object: Type[Pinecone], params: dict): """Initialize pinecone and return the class object""" diff --git a/src/backend/langflow/template/frontend_node/vectorstores.py b/src/backend/langflow/template/frontend_node/vectorstores.py index 5e25d3403..ce2c35434 100644 --- a/src/backend/langflow/template/frontend_node/vectorstores.py +++ b/src/backend/langflow/template/frontend_node/vectorstores.py @@ -18,7 +18,18 @@ class VectorStoreFrontendNode(FrontendNode): multiline=False, value="http://localhost:8080", ) - extra_fields.append(extra_field) + # Add client_kwargs field + extra_field2 = TemplateField( + name="client_kwargs", + field_type="code", + required=False, + placeholder="", + show=True, + advanced=True, + multiline=False, + value="{}", + ) + extra_fields.extend((extra_field, extra_field2)) elif self.template.type_name == "Chroma": # New bool field for persist parameter @@ -55,6 +66,29 @@ class VectorStoreFrontendNode(FrontendNode): value="", ) extra_fields.extend((extra_field, extra_field2)) + elif self.template.type_name == "FAISS": + extra_field = TemplateField( + name="folder_path", + field_type="str", + required=False, + placeholder="", + show=True, + advanced=True, + multiline=False, + display_name="Local Path", + value="", + ) + extra_field2 = TemplateField( + name="index_name", + field_type="str", + required=False, + show=True, + advanced=False, + value="", + display_name="Index Name", + ) + extra_fields.extend((extra_field, extra_field2)) + if extra_fields: for field in extra_fields: self.template.add_field(field) @@ -76,6 +110,7 @@ class VectorStoreFrontendNode(FrontendNode): "weaviate_url", "index_name", "namespace", + "folder_path", ] advanced_fields = [ "n_dim", @@ -94,6 +129,7 @@ class VectorStoreFrontendNode(FrontendNode): "grpc_port", "pinecone_api_key", "pinecone_env", + "client_kwargs", ] # Check and set field attributes