Change docstring parser (#158)

Using library `docstring-parser`

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -776,6 +776,18 @@ files = [
 [package.extras]
 graph = ["objgraph (>=1.7.2)"]
 
+[[package]]
+name = "docstring-parser"
+version = "0.15"
+description = "Parse Python docstrings in reST, Google and Numpydoc format"
+category = "main"
+optional = false
+python-versions = ">=3.6,<4.0"
+files = [
+    {file = "docstring_parser-0.15-py3-none-any.whl", hash = "sha256:d1679b86250d269d06a99670924d6bce45adc00b08069dae8c47d98e89b667a9"},
+    {file = "docstring_parser-0.15.tar.gz", hash = "sha256:48ddc093e8b1865899956fcc03b03e66bb7240c310fac5af81814580c55bf682"},
+]
+
 [[package]]
 name = "duckdb"
 version = "0.7.1"
@@ -1584,14 +1596,14 @@ files = [
 
 [[package]]
 name = "jupyter-client"
-version = "8.1.0"
+version = "8.2.0"
 description = "Jupyter protocol implementation and client libraries"
 category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "jupyter_client-8.1.0-py3-none-any.whl", hash = "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"},
-    {file = "jupyter_client-8.1.0.tar.gz", hash = "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811"},
+    {file = "jupyter_client-8.2.0-py3-none-any.whl", hash = "sha256:b18219aa695d39e2ad570533e0d71fb7881d35a873051054a84ee2a17c4b7389"},
+    {file = "jupyter_client-8.2.0.tar.gz", hash = "sha256:9fe233834edd0e6c0aa5f05ca2ab4bdea1842bfd2d8a932878212fc5301ddaf0"},
 ]
 
 [package.dependencies]
@@ -1604,7 +1616,7 @@ traitlets = ">=5.3"
 
 [package.extras]
 docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"]
-test = ["codecov", "coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
+test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
 
 [[package]]
 name = "jupyter-core"
@@ -3894,6 +3906,10 @@ category = "main"
 optional = false
 python-versions = ">=3.8.0"
 files = [
+    {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"},
+    {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"},
+    {file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"},
+    {file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"},
     {file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"},
     {file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"},
     {file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"},
@@ -4111,6 +4127,15 @@ category = "main"
 optional = false
 python-versions = "*"
 files = [
+    {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"},
+    {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"},
+    {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"},
+    {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"},
+    {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"},
+    {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"},
+    {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"},
+    {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"},
+    {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"},
     {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"},
     {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"},
     {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"},
@@ -4687,4 +4712,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "4f27ad94f244998e9e79fe1ae733cc786d82025f022c1d337e5c84d0393947b6"
+content-hash = "313af2197643e0cbac777d28d0a996e1f61f7c1f809a375b5196cc5942c5cc4b"

pyproject.toml

@@ -44,6 +44,7 @@ pypdf = "^3.7.1"
 lxml = "^4.9.2"
 pysrt = "^1.1.2"
 fake-useragent = "^1.1.3"
+docstring-parser = "^0.15"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"

src/backend/langflow/graph/graph.py

@@ -27,7 +27,7 @@ from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
 from langflow.interface.tools.constants import FILE_TOOLS
-from langflow.interface.vectorstore.base import vectorstore_creator
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 from langflow.utils import payload
 

src/backend/langflow/interface/base.py

@@ -1,10 +1,10 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Type, Union
-from langflow.utils.logger import logger
 
 from pydantic import BaseModel
 
 from langflow.template.base import FrontendNode, Template, TemplateField
+from langflow.utils.logger import logger
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 

src/backend/langflow/interface/chains/base.py

@@ -5,8 +5,8 @@ from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import chain_type_to_cls_dict
 from langflow.settings import settings
 from langflow.template.nodes import ChainFrontendNode
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 # Assuming necessary imports for Field, Template, and FrontendNode classes
 

src/backend/langflow/interface/document_loaders/base.py

@@ -3,8 +3,8 @@ from typing import Dict, List, Optional
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import documentloaders_type_to_cls_dict
 from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 def build_file_path_template(

src/backend/langflow/interface/embeddings/base.py

@@ -3,8 +3,8 @@ from typing import Dict, List, Optional
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import embedding_type_to_cls_dict
 from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class EmbeddingCreator(LangChainTypeCreator):

src/backend/langflow/interface/listing.py

@@ -8,7 +8,7 @@ from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
-from langflow.interface.vectorstore.base import vectorstore_creator
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
 

src/backend/langflow/interface/llms/base.py

@@ -4,8 +4,8 @@ from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import llm_type_to_cls_dict
 from langflow.settings import settings
 from langflow.template.nodes import LLMFrontendNode
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class LLMCreator(LangChainTypeCreator):

src/backend/langflow/interface/prompts/base.py

@@ -7,8 +7,8 @@ from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.importing.utils import import_class
 from langflow.settings import settings
 from langflow.template.nodes import PromptFrontendNode
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class PromptCreator(LangChainTypeCreator):

src/backend/langflow/interface/toolkits/base.py

@@ -5,8 +5,8 @@ from langchain.agents import agent_toolkits
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.importing.utils import import_class, import_module
 from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class ToolkitCreator(LangChainTypeCreator):

src/backend/langflow/interface/types.py

@@ -8,7 +8,7 @@ from langflow.interface.prompts.base import prompt_creator
 from langflow.interface.text_splitters.base import textsplitter_creator
 from langflow.interface.toolkits.base import toolkits_creator
 from langflow.interface.tools.base import tool_creator
-from langflow.interface.vectorstore.base import vectorstore_creator
+from langflow.interface.vector_store.base import vectorstore_creator
 from langflow.interface.wrappers.base import wrapper_creator
 
 

src/backend/langflow/interface/vector_store/base.py

@@ -3,8 +3,8 @@ from typing import Dict, List, Optional
 from langflow.interface.base import LangChainTypeCreator
 from langflow.interface.custom_lists import vectorstores_type_to_cls_dict
 from langflow.settings import settings
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class VectorstoreCreator(LangChainTypeCreator):

src/backend/langflow/interface/wrappers/base.py

@@ -3,8 +3,8 @@ from typing import Dict, List, Optional
 from langchain import requests
 
 from langflow.interface.base import LangChainTypeCreator
-from langflow.utils.util import build_template_from_class
 from langflow.utils.logger import logger
+from langflow.utils.util import build_template_from_class
 
 
 class WrapperCreator(LangChainTypeCreator):

src/backend/langflow/utils/util.py

@@ -3,6 +3,8 @@ import inspect
 import re
 from typing import Dict, Optional
 
+from docstring_parser import parse  # type: ignore
+
 from langflow.template.constants import FORCE_SHOW_FIELDS
 from langflow.utils import constants
 
@@ -65,7 +67,8 @@ def build_template_from_function(
         if v.__annotations__["return"].__name__ == name:
             _class = v.__annotations__["return"]
 
-            docs = get_class_doc(_class)
+            # Get the docstring
+            docs = parse(_class.__doc__)
 
             variables = {"_type": _type}
             for class_field_items, value in _class.__fields__.items():
@@ -86,8 +89,8 @@ def build_template_from_function(
                         variables[class_field_items][name_] = value_
 
                 variables[class_field_items]["placeholder"] = (
-                    docs["Attributes"][class_field_items]
-                    if class_field_items in docs["Attributes"]
+                    docs.params[class_field_items]
+                    if class_field_items in docs.params
                     else ""
                 )
             # Adding function to base classes to allow
@@ -98,7 +101,7 @@ def build_template_from_function(
 
             return {
                 "template": format_dict(variables, name),
-                "description": docs["Description"],
+                "description": docs.short_description or "",
                 "base_classes": base_classes,
             }
 
@@ -117,7 +120,7 @@ def build_template_from_class(
             _class = v
 
             # Get the docstring
-            docs = get_class_doc(_class)
+            docs = parse(_class.__doc__)
 
             variables = {"_type": _type}
 
@@ -140,8 +143,8 @@ def build_template_from_class(
                             variables[class_field_items][name_] = value_
 
                     variables[class_field_items]["placeholder"] = (
-                        docs["Attributes"][class_field_items]
-                        if class_field_items in docs["Attributes"]
+                        docs.params[class_field_items]
+                        if class_field_items in docs.params
                         else ""
                     )
             base_classes = get_base_classes(_class)
@@ -151,7 +154,7 @@ def build_template_from_class(
                 base_classes.append("function")
             return {
                 "template": format_dict(variables, name),
-                "description": docs["Description"],
+                "description": docs.short_description or "",
                 "base_classes": base_classes,
             }
 
@@ -188,60 +191,6 @@ def get_default_factory(module: str, function: str):
     return None
 
 
-def get_class_doc(class_name):
-    """
-    Extracts information from the docstring of a given class.
-
-    Args:
-        class_name: the class to extract information from
-
-    Returns:
-        A dictionary containing the extracted information, with keys
-        for 'Description', 'Parameters', 'Attributes', and 'Returns'.
-    """
-    # Template
-    data = {
-        "Description": "",
-        "Parameters": {},
-        "Attributes": {},
-        "Example": [],
-        "Returns": {},
-    }
-
-    # Get the class docstring
-    docstring = class_name.__doc__
-
-    if not docstring:
-        return data
-
-    # Parse the docstring to extract information
-    lines = docstring.split("\n")
-
-    current_section = "Description"
-
-    for line in lines:
-        line = line.strip()
-
-        if not line:
-            continue
-
-        if (
-            line.startswith(tuple(data.keys()))
-            and len(line.split()) == 1
-            and line.endswith(":")
-        ):
-            current_section = line[:-1]
-            continue
-
-        if current_section in ["Description", "Example"]:
-            data[current_section] += line
-        else:
-            param, desc = line.split(":")
-            data[current_section][param.strip()] = desc.strip()
-
-    return data
-
-
 def format_dict(d, name: Optional[str] = None):
     """
     Formats a dictionary by removing certain keys and modifying the

tests/test_llms_template.py

@@ -440,7 +440,7 @@ def test_chat_open_ai(client: TestClient):
     assert template["_type"] == "ChatOpenAI"
     assert (
         model["description"]
-        == "Wrapper around OpenAI Chat large language models.To use, you should have the ``openai`` python package installed, and theenvironment variable ``OPENAI_API_KEY`` set with your API key.Any parameters that are valid to be passed to the openai.create call can be passedin, even if not explicitly saved on this class."  # noqa E501
+        == "Wrapper around OpenAI Chat large language models."  # noqa E501
     )
     assert set(model["base_classes"]) == {
         "BaseChatModel",

tests/test_template.py

@@ -8,7 +8,6 @@ from langflow.utils.util import (
     build_template_from_function,
     format_dict,
     get_base_classes,
-    get_class_doc,
     get_default_factory,
 )
 from pydantic import BaseModel
@@ -280,12 +279,3 @@ def test_get_default_factory():
     default_value = get_default_factory(module_name, function_repr)
 
     assert default_value == "default_value"
-
-
-# Test get_class_doc
-def test_get_class_doc():
-    class_doc_parent = get_class_doc(Parent)
-    class_doc_child = get_class_doc(Child)
-
-    assert class_doc_parent["Description"] == "Parent Class"
-    assert class_doc_child["Description"] == "Child Class"