From ba1f7daa7af12fc0a88ff6c0f8146b0fafecf71b Mon Sep 17 00:00:00 2001 From: Ibis Prevedello Date: Thu, 13 Apr 2023 21:39:25 -0300 Subject: [PATCH] refac: change docstring parser --- poetry.lock | 37 ++++++++++++--- pyproject.toml | 1 + src/backend/langflow/utils/util.py | 73 +++++------------------------- tests/test_template.py | 10 ---- 4 files changed, 43 insertions(+), 78 deletions(-) diff --git a/poetry.lock b/poetry.lock index 71ce345dc..27ade8589 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -776,6 +776,18 @@ files = [ [package.extras] graph = ["objgraph (>=1.7.2)"] +[[package]] +name = "docstring-parser" +version = "0.15" +description = "Parse Python docstrings in reST, Google and Numpydoc format" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "docstring_parser-0.15-py3-none-any.whl", hash = "sha256:d1679b86250d269d06a99670924d6bce45adc00b08069dae8c47d98e89b667a9"}, + {file = "docstring_parser-0.15.tar.gz", hash = "sha256:48ddc093e8b1865899956fcc03b03e66bb7240c310fac5af81814580c55bf682"}, +] + [[package]] name = "duckdb" version = "0.7.1" @@ -1584,14 +1596,14 @@ files = [ [[package]] name = "jupyter-client" -version = "8.1.0" +version = "8.2.0" description = "Jupyter protocol implementation and client libraries" category = "dev" optional = false python-versions = ">=3.8" files = [ - {file = "jupyter_client-8.1.0-py3-none-any.whl", hash = "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"}, - {file = "jupyter_client-8.1.0.tar.gz", hash = "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811"}, + {file = "jupyter_client-8.2.0-py3-none-any.whl", hash = "sha256:b18219aa695d39e2ad570533e0d71fb7881d35a873051054a84ee2a17c4b7389"}, + {file = "jupyter_client-8.2.0.tar.gz", hash = "sha256:9fe233834edd0e6c0aa5f05ca2ab4bdea1842bfd2d8a932878212fc5301ddaf0"}, ] [package.dependencies] @@ -1604,7 +1616,7 @@ traitlets = ">=5.3" [package.extras] docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["codecov", "coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] [[package]] name = "jupyter-core" @@ -3894,6 +3906,10 @@ category = "main" optional = false python-versions = ">=3.8.0" files = [ + {file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"}, + {file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"}, + {file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"}, + {file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"}, {file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"}, {file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"}, {file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"}, @@ -4111,6 +4127,15 @@ category = "main" optional = false python-versions = "*" files = [ + {file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"}, + {file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"}, + {file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"}, + {file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"}, + {file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"}, + {file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"}, + {file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"}, + {file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"}, + {file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"}, {file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"}, {file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"}, {file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"}, @@ -4687,4 +4712,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "4f27ad94f244998e9e79fe1ae733cc786d82025f022c1d337e5c84d0393947b6" +content-hash = "313af2197643e0cbac777d28d0a996e1f61f7c1f809a375b5196cc5942c5cc4b" diff --git a/pyproject.toml b/pyproject.toml index 6ccc563aa..b069afc3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ pypdf = "^3.7.1" lxml = "^4.9.2" pysrt = "^1.1.2" fake-useragent = "^1.1.3" +docstring-parser = "^0.15" [tool.poetry.group.dev.dependencies] black = "^23.1.0" diff --git a/src/backend/langflow/utils/util.py b/src/backend/langflow/utils/util.py index 1ca73746b..b31a3bed1 100644 --- a/src/backend/langflow/utils/util.py +++ b/src/backend/langflow/utils/util.py @@ -3,6 +3,8 @@ import inspect import re from typing import Dict, Optional +from docstring_parser import parse # type: ignore + from langflow.template.constants import FORCE_SHOW_FIELDS from langflow.utils import constants @@ -65,7 +67,8 @@ def build_template_from_function( if v.__annotations__["return"].__name__ == name: _class = v.__annotations__["return"] - docs = get_class_doc(_class) + # Get the docstring + docs = parse(_class.__doc__) variables = {"_type": _type} for class_field_items, value in _class.__fields__.items(): @@ -86,8 +89,8 @@ def build_template_from_function( variables[class_field_items][name_] = value_ variables[class_field_items]["placeholder"] = ( - docs["Attributes"][class_field_items] - if class_field_items in docs["Attributes"] + docs.params[class_field_items] + if class_field_items in docs.params else "" ) # Adding function to base classes to allow @@ -98,7 +101,7 @@ def build_template_from_function( return { "template": format_dict(variables, name), - "description": docs["Description"], + "description": docs.short_description or "", "base_classes": base_classes, } @@ -117,7 +120,7 @@ def build_template_from_class( _class = v # Get the docstring - docs = get_class_doc(_class) + docs = parse(_class.__doc__) variables = {"_type": _type} @@ -140,8 +143,8 @@ def build_template_from_class( variables[class_field_items][name_] = value_ variables[class_field_items]["placeholder"] = ( - docs["Attributes"][class_field_items] - if class_field_items in docs["Attributes"] + docs.params[class_field_items] + if class_field_items in docs.params else "" ) base_classes = get_base_classes(_class) @@ -151,7 +154,7 @@ def build_template_from_class( base_classes.append("function") return { "template": format_dict(variables, name), - "description": docs["Description"], + "description": docs.short_description or "", "base_classes": base_classes, } @@ -188,60 +191,6 @@ def get_default_factory(module: str, function: str): return None -def get_class_doc(class_name): - """ - Extracts information from the docstring of a given class. - - Args: - class_name: the class to extract information from - - Returns: - A dictionary containing the extracted information, with keys - for 'Description', 'Parameters', 'Attributes', and 'Returns'. - """ - # Template - data = { - "Description": "", - "Parameters": {}, - "Attributes": {}, - "Example": [], - "Returns": {}, - } - - # Get the class docstring - docstring = class_name.__doc__ - - if not docstring: - return data - - # Parse the docstring to extract information - lines = docstring.split("\n") - - current_section = "Description" - - for line in lines: - line = line.strip() - - if not line: - continue - - if ( - line.startswith(tuple(data.keys())) - and len(line.split()) == 1 - and line.endswith(":") - ): - current_section = line[:-1] - continue - - if current_section in ["Description", "Example"]: - data[current_section] += line - else: - param, desc = line.split(":") - data[current_section][param.strip()] = desc.strip() - - return data - - def format_dict(d, name: Optional[str] = None): """ Formats a dictionary by removing certain keys and modifying the diff --git a/tests/test_template.py b/tests/test_template.py index b5a424dab..a9b5a71ed 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -8,7 +8,6 @@ from langflow.utils.util import ( build_template_from_function, format_dict, get_base_classes, - get_class_doc, get_default_factory, ) from pydantic import BaseModel @@ -280,12 +279,3 @@ def test_get_default_factory(): default_value = get_default_factory(module_name, function_repr) assert default_value == "default_value" - - -# Test get_class_doc -def test_get_class_doc(): - class_doc_parent = get_class_doc(Parent) - class_doc_child = get_class_doc(Child) - - assert class_doc_parent["Description"] == "Parent Class" - assert class_doc_child["Description"] == "Child Class"