From 1a0d448e82a2dddb2b2ab1c1216a66c6d2863082 Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Thu, 22 Jun 2023 13:51:18 -0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20feat(validate.py):=20remove=20un?= =?UTF-8?q?used=20imports=20and=20endpoint=20for=20node=20validation=20?= =?UTF-8?q?=F0=9F=9A=80=20feat(base.py):=20update=20import=20statement=20f?= =?UTF-8?q?or=20loading=20module=20=F0=9F=9A=80=20feat(vector=5Fstore.py):?= =?UTF-8?q?=20add=20initialization=20functions=20for=20pinecone,=20chroma,?= =?UTF-8?q?=20and=20qdrant=20The=20unused=20imports=20and=20endpoint=20for?= =?UTF-8?q?=20node=20validation=20were=20removed=20from=20validate.py.=20T?= =?UTF-8?q?he=20import=20statement=20for=20the=20loading=20module=20was=20?= =?UTF-8?q?updated=20in=20base.py.=20Initialization=20functions=20for=20pi?= =?UTF-8?q?necone,=20chroma,=20and=20qdrant=20were=20added=20to=20vector?= =?UTF-8?q?=5Fstore.py=20to=20allow=20for=20the=20creation=20of=20objects?= =?UTF-8?q?=20from=20the=20respective=20services.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/api/v1/validate.py | 27 ------- src/backend/langflow/graph/vertex/base.py | 2 +- .../interface/initialize/vector_store.py | 75 +++++++++++++++++++ 3 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 src/backend/langflow/interface/initialize/vector_store.py diff --git a/src/backend/langflow/api/v1/validate.py b/src/backend/langflow/api/v1/validate.py index d7ba8c15e..959273a00 100644 --- a/src/backend/langflow/api/v1/validate.py +++ b/src/backend/langflow/api/v1/validate.py @@ -1,5 +1,3 @@ -import json - from fastapi import APIRouter, HTTPException from langflow.api.v1.base import ( @@ -9,8 +7,6 @@ from langflow.api.v1.base import ( PromptValidationResponse, validate_prompt, ) -from langflow.graph.vertex.types import VectorStoreVertex -from langflow.graph import Graph from langflow.utils.logger import logger from langflow.utils.validate import validate_code @@ -37,26 +33,3 @@ def post_validate_prompt(prompt: Prompt): except Exception as e: logger.exception(e) raise HTTPException(status_code=500, detail=str(e)) from e - - -# validate node -@router.post("/node/{node_id}", status_code=200) -def post_validate_node(node_id: str, data: dict): - try: - # build graph - graph = Graph.from_payload(data) - # validate node - node = graph.get_node(node_id) - if node is None: - raise ValueError(f"Node {node_id} not found") - if not isinstance(node, VectorStoreVertex): - node.build() - return json.dumps( - { - "valid": True, - "params": f"{str(node._built_object_repr())[:300]}...", - } - ) - except Exception as e: - logger.exception(e) - return json.dumps({"valid": False, "params": str(e)}) diff --git a/src/backend/langflow/graph/vertex/base.py b/src/backend/langflow/graph/vertex/base.py index a86e51d25..87a09d604 100644 --- a/src/backend/langflow/graph/vertex/base.py +++ b/src/backend/langflow/graph/vertex/base.py @@ -1,5 +1,5 @@ from langflow.graph.vertex.constants import DIRECT_TYPES -from langflow.interface import loading +from langflow.interface.initialize import loading from langflow.interface.listing import ALL_TYPES_DICT from langflow.utils.logger import logger from langflow.utils.util import sync_to_async diff --git a/src/backend/langflow/interface/initialize/vector_store.py b/src/backend/langflow/interface/initialize/vector_store.py new file mode 100644 index 000000000..f3a10d0ba --- /dev/null +++ b/src/backend/langflow/interface/initialize/vector_store.py @@ -0,0 +1,75 @@ +from typing import Any + + +def docs_in_params(params: dict) -> bool: + """Check if params has documents OR texts and one of them is not an empty list""" + return ("documents" in params and params["documents"] != []) or ( + "texts" in params and params["texts"] != [] + ) + + +def initialize_pinecone(class_object: Any, params: dict): + """Initialize pinecone and return the class object""" + PINECONE_API_KEY = params.get("pinecone_api_key") + PINECONE_ENV = params.get("pinecone_env") + + if PINECONE_API_KEY is None or PINECONE_ENV is None: + raise ValueError( + "Pinecone API key and environment must be provided in the params" + ) + + if not docs_in_params(params): + import pinecone + + # initialize pinecone + pinecone.init( + api_key=PINECONE_API_KEY, # find at app.pinecone.io + environment=PINECONE_ENV, # next to api key in console + ) + # If there are docs in the index, delete them + return class_object.from_existing_index(**params) + # If there are docs in the params, create a new index + if "texts" in params: + params["documents"] = params.pop("texts") + return class_object.from_documents(**params) + + +def initialize_chroma(class_object, params): + """Initialize a ChromaDB object from the params""" + persist = params.pop("persist", False) + if not docs_in_params(params): + if "texts" in params: + params["documents"] = params.pop("texts") + for doc in params["documents"]: + if doc.metadata is None: + doc.metadata = {} + for key, value in doc.metadata.items(): + if value is None: + doc.metadata[key] = "" + chromadb = class_object.from_documents(**params) + else: + chromadb = class_object(**params) + if persist: + chromadb.persist() + return chromadb + + +def initialize_qdrant(class_object, params): + if not docs_in_params(params): + if "location" not in params and "api_key" not in params: + raise ValueError("Location and API key must be provided in the params") + from qdrant_client import QdrantClient + + client_params = { + "location": params.pop("location"), + "api_key": params.pop("api_key"), + } + lc_params = { + "collection_name": params.pop("collection_name"), + "embeddings": params.pop("embedding"), + } + client = QdrantClient(**client_params) + + return class_object(client=client, **lc_params) + + return class_object.from_documents(**params)