Adds Vectara VectorStore and Metaphor Toolkit (#743)

poetry.lock

@@ -2,13 +2,13 @@
 
 [[package]]
 name = "aiofiles"
-version = "23.2.0"
+version = "23.2.1"
 description = "File support for asyncio."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "aiofiles-23.2.0-py3-none-any.whl", hash = "sha256:d7adbeef4bada163e70bd3163866680332f622b4437d9d8246bd74b6d791d83a"},
-    {file = "aiofiles-23.2.0.tar.gz", hash = "sha256:7a321e787cb3839e7d70f56121bf9197a1980609a0fe06a6f96c573406ce4fca"},
+    {file = "aiofiles-23.2.1-py3-none-any.whl", hash = "sha256:19297512c647d4b27a2cf7c34caa7e405c0d60b5560618a29a9fe027b18b0107"},
+    {file = "aiofiles-23.2.1.tar.gz", hash = "sha256:84ec2218d8419404abcb9f0c02df3f34c6e0a68ed41072acfb1cef5cbc29051a"},
 ]
 
 [[package]]
@@ -1065,13 +1065,13 @@ files = [
 
 [[package]]
 name = "dnspython"
-version = "2.4.1"
+version = "2.4.2"
 description = "DNS toolkit"
 optional = false
 python-versions = ">=3.8,<4.0"
 files = [
-    {file = "dnspython-2.4.1-py3-none-any.whl", hash = "sha256:5b7488477388b8c0b70a8ce93b227c5603bc7b77f1565afe8e729c36c51447d7"},
-    {file = "dnspython-2.4.1.tar.gz", hash = "sha256:c33971c79af5be968bb897e95c2448e11a645ee84d93b265ce0b7aabe5dfdca8"},
+    {file = "dnspython-2.4.2-py3-none-any.whl", hash = "sha256:57c6fbaaeaaf39c891292012060beb141791735dbb4004798328fc2c467402d8"},
+    {file = "dnspython-2.4.2.tar.gz", hash = "sha256:8dcfae8c7460a2f84b4072e26f1c9f4101ca20c071649cb7c34e8b6a93d58984"},
 ]
 
 [package.extras]
@@ -3315,6 +3315,20 @@ files = [
     {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
 ]
 
+[[package]]
+name = "metaphor-python"
+version = "0.1.11"
+description = "A Python package for the Metaphor API."
+optional = false
+python-versions = "*"
+files = [
+    {file = "metaphor-python-0.1.11.tar.gz", hash = "sha256:80fd993c44cc9d453d99eb65b95147d305f542fdd6fda699e3852e3100beb6ec"},
+    {file = "metaphor_python-0.1.11-py3-none-any.whl", hash = "sha256:0d759ecdf73492a4bafd404d0444935c172bcc4a89334f4f6780863ba488b238"},
+]
+
+[package.dependencies]
+requests = "*"
+
 [[package]]
 name = "monotonic"
 version = "1.6"
@@ -7579,4 +7593,4 @@ local = ["ctransformers", "llama-cpp-python", "sentence-transformers"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.11"
-content-hash = "cfce7822aedd150c0bda5f0dbf4ada3a3d80d0a7d968c081a5dfd9a61d5088ad"
+content-hash = "53bb67a463c4ad0d3dd30c89e30750fc31be5530ee54539d4df34fd82ecf7bc1"

pyproject.toml

@@ -78,6 +78,7 @@ psycopg = "^3.1.9"
 psycopg-binary = "^3.1.9"
 fastavro = "^1.8.0"
 langchain-experimental = "^0.0.8"
+metaphor-python = "^0.1.11"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"

src/backend/langflow/components/toolkits/Metaphor.py

@@ -0,0 +1,56 @@
+from typing import List, Union
+from langflow import CustomComponent
+
+from metaphor_python import Metaphor  # type: ignore
+from langchain.tools import Tool
+from langchain.agents import tool
+from langchain.agents.agent_toolkits.base import BaseToolkit
+
+
+class MetaphorToolkit(CustomComponent):
+    display_name: str = "Metaphor"
+    description: str = "Metaphor Toolkit"
+    documentation = (
+        "https://python.langchain.com/docs/integrations/tools/metaphor_search"
+    )
+    beta = True
+    # api key should be password = True
+    field_config = {
+        "metaphor_api_key": {"display_name": "Metaphor API Key", "password": True},
+        "code": {"advanced": True},
+    }
+
+    def build(
+        self,
+        metaphor_api_key: str,
+        use_autoprompt: bool = True,
+        search_num_results: int = 5,
+        similar_num_results: int = 5,
+    ) -> Union[Tool, BaseToolkit]:
+        # If documents, then we need to create a Vectara instance using .from_documents
+        client = Metaphor(api_key=metaphor_api_key)
+
+        @tool
+        def search(query: str):
+            """Call search engine with a query."""
+            return client.search(
+                query, use_autoprompt=use_autoprompt, num_results=search_num_results
+            )
+
+        @tool
+        def get_contents(ids: List[str]):
+            """Get contents of a webpage.
+
+            The ids passed in should be a list of ids as fetched from `search`.
+            """
+            return client.get_contents(ids)
+
+        @tool
+        def find_similar(url: str):
+            """Get search results similar to a given URL.
+
+            The url passed in should be a URL returned from `search`
+            """
+            return client.find_similar(url, num_results=similar_num_results)
+
+        return [search, get_contents, find_similar]  # type: ignore

src/backend/langflow/components/vectorstores/Vectara.py

@@ -0,0 +1,50 @@
+from typing import Optional, Union
+from langflow import CustomComponent
+
+from langchain.vectorstores import Vectara
+from langchain.schema import Document
+from langchain.vectorstores.base import VectorStore
+from langchain.schema import BaseRetriever
+from langchain.embeddings.base import Embeddings
+
+
+class VectaraComponent(CustomComponent):
+    display_name: str = "Vectara"
+    description: str = "Implementation of Vector Store using Vectara"
+    documentation = (
+        "https://python.langchain.com/docs/integrations/vectorstores/vectara"
+    )
+    beta = True
+    # api key should be password = True
+    field_config = {
+        "vectara_customer_id": {"display_name": "Vectara Customer ID"},
+        "vectara_corpus_id": {"display_name": "Vectara Corpus ID"},
+        "vectara_api_key": {"display_name": "Vectara API Key", "password": True},
+        "code": {"show": False},
+        "documents": {"display_name": "Documents"},
+        "embedding": {"display_name": "Embedding"},
+    }
+
+    def build(
+        self,
+        vectara_customer_id: str,
+        vectara_corpus_id: str,
+        vectara_api_key: str,
+        embedding: Optional[Embeddings] = None,
+        documents: Optional[Document] = None,
+    ) -> Union[VectorStore, BaseRetriever]:
+        # If documents, then we need to create a Vectara instance using .from_documents
+        if documents is not None and embedding is not None:
+            return Vectara.from_documents(
+                documents=documents,  # type: ignore
+                vectara_customer_id=vectara_customer_id,
+                vectara_corpus_id=vectara_corpus_id,
+                vectara_api_key=vectara_api_key,
+                embedding=embedding,
+            )
+
+        return Vectara(
+            vectara_customer_id=vectara_customer_id,
+            vectara_corpus_id=vectara_corpus_id,
+            vectara_api_key=vectara_api_key,
+        )

src/backend/langflow/interface/custom/component.py

@@ -66,6 +66,9 @@ class Component(BaseModel):
                 elif "beta" in item_name:
                     template_config["beta"] = ast.literal_eval(item_value)
 
+                elif "documentation" in item_name:
+                    template_config["documentation"] = ast.literal_eval(item_value)
+
         return template_config
 
     def build(self, *args: Any, **kwargs: Any) -> Any:

src/backend/langflow/interface/custom/custom_component.py

@@ -92,9 +92,9 @@ class CustomComponent(Component, extra=Extra.allow):
         return build_method["args"]
 
     @property
-    def get_function_entrypoint_return_type(self) -> str:
+    def get_function_entrypoint_return_type(self) -> List[str]:
         if not self.code:
-            return ""
+            return []
         tree = self.get_code_tree(self.code)
 
         component_classes = [
@@ -103,7 +103,7 @@ class CustomComponent(Component, extra=Extra.allow):
             if self.code_class_base_inheritance in cls["bases"]
         ]
         if not component_classes:
-            return ""
+            return []
 
         # Assume the first Component class is the one we're interested in
         component_class = component_classes[0]
@@ -114,11 +114,19 @@ class CustomComponent(Component, extra=Extra.allow):
         ]
 
         if not build_methods:
-            return ""
+            return []
 
         build_method = build_methods[0]
+        return_type = build_method["return_type"]
+        # If the return type is not a Union, then we just return it as a list
+        if "Union" not in return_type:
+            return [return_type] if return_type in self.return_type_valid_list else []
 
-        return build_method["return_type"]
+        # If the return type is a Union, then we need to parse it
+        return_type = return_type.replace("Union", "").replace("[", "").replace("]", "")
+        return_type = return_type.split(",")
+        return_type = [item.strip() for item in return_type]
+        return [item for item in return_type if item in self.return_type_valid_list]
 
     @property
     def get_main_class_name(self):

src/backend/langflow/interface/types.py

@@ -1,6 +1,6 @@
 import ast
 import contextlib
-from typing import Any
+from typing import Any, List
 from langflow.api.utils import merge_nested_dicts_with_renaming
 from langflow.interface.agents.base import agent_creator
 from langflow.interface.chains.base import chain_creator
@@ -199,6 +199,9 @@ def update_attributes(frontend_node, template_config):
     if "beta" in template_config:
         frontend_node["beta"] = template_config["beta"]
 
+    if "documentation" in template_config:
+        frontend_node["documentation"] = template_config["documentation"]
+
 
 def build_field_config(custom_component: CustomComponent):
     """Build the field configuration for a custom component"""
@@ -257,26 +260,27 @@ def get_field_properties(extra_field):
     return field_name, field_type, field_value, field_required
 
 
-def add_base_classes(frontend_node, return_type):
+def add_base_classes(frontend_node, return_types: List[str]):
     """Add base classes to the frontend node"""
-    if return_type not in CUSTOM_COMPONENT_SUPPORTED_TYPES or return_type is None:
-        raise HTTPException(
-            status_code=400,
-            detail={
-                "error": (
-                    "Invalid return type should be one of: "
-                    f"{list(CUSTOM_COMPONENT_SUPPORTED_TYPES.keys())}"
-                ),
-                "traceback": traceback.format_exc(),
-            },
-        )
+    for return_type in return_types:
+        if return_type not in CUSTOM_COMPONENT_SUPPORTED_TYPES or return_type is None:
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": (
+                        "Invalid return type should be one of: "
+                        f"{list(CUSTOM_COMPONENT_SUPPORTED_TYPES.keys())}"
+                    ),
+                    "traceback": traceback.format_exc(),
+                },
+            )
 
-    return_type_instance = CUSTOM_COMPONENT_SUPPORTED_TYPES.get(return_type)
-    base_classes = get_base_classes(return_type_instance)
+        return_type_instance = CUSTOM_COMPONENT_SUPPORTED_TYPES.get(return_type)
+        base_classes = get_base_classes(return_type_instance)
 
-    for base_class in base_classes:
-        if base_class not in CLASSES_TO_REMOVE:
-            frontend_node.get("base_classes").append(base_class)
+        for base_class in base_classes:
+            if base_class not in CLASSES_TO_REMOVE:
+                frontend_node.get("base_classes").append(base_class)
 
 
 def build_langchain_template_custom_component(custom_component: CustomComponent):