refac: change docstring parser
This commit is contained in:
Ibis Prevedello
parent
ce542e9337
commit
ba1f7daa7a
4 changed files with 43 additions and 78 deletions
37
poetry.lock
generated
37
poetry.lock
generated
|
|
@ -1,4 +1,4 @@
|
|||
# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
|
||||
# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "aiohttp"
|
||||
|
|
@ -776,6 +776,18 @@ files = [
|
|||
[package.extras]
|
||||
graph = ["objgraph (>=1.7.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "docstring-parser"
|
||||
version = "0.15"
|
||||
description = "Parse Python docstrings in reST, Google and Numpydoc format"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6,<4.0"
|
||||
files = [
|
||||
{file = "docstring_parser-0.15-py3-none-any.whl", hash = "sha256:d1679b86250d269d06a99670924d6bce45adc00b08069dae8c47d98e89b667a9"},
|
||||
{file = "docstring_parser-0.15.tar.gz", hash = "sha256:48ddc093e8b1865899956fcc03b03e66bb7240c310fac5af81814580c55bf682"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "duckdb"
|
||||
version = "0.7.1"
|
||||
|
|
@ -1584,14 +1596,14 @@ files = [
|
|||
|
||||
[[package]]
|
||||
name = "jupyter-client"
|
||||
version = "8.1.0"
|
||||
version = "8.2.0"
|
||||
description = "Jupyter protocol implementation and client libraries"
|
||||
category = "dev"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
files = [
|
||||
{file = "jupyter_client-8.1.0-py3-none-any.whl", hash = "sha256:d5b8e739d7816944be50f81121a109788a3d92732ecf1ad1e4dadebc948818fe"},
|
||||
{file = "jupyter_client-8.1.0.tar.gz", hash = "sha256:3fbab64100a0dcac7701b1e0f1a4412f1ccb45546ff2ad9bc4fcbe4e19804811"},
|
||||
{file = "jupyter_client-8.2.0-py3-none-any.whl", hash = "sha256:b18219aa695d39e2ad570533e0d71fb7881d35a873051054a84ee2a17c4b7389"},
|
||||
{file = "jupyter_client-8.2.0.tar.gz", hash = "sha256:9fe233834edd0e6c0aa5f05ca2ab4bdea1842bfd2d8a932878212fc5301ddaf0"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
|
|
@ -1604,7 +1616,7 @@ traitlets = ">=5.3"
|
|||
|
||||
[package.extras]
|
||||
docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"]
|
||||
test = ["codecov", "coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
|
||||
test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"]
|
||||
|
||||
[[package]]
|
||||
name = "jupyter-core"
|
||||
|
|
@ -3894,6 +3906,10 @@ category = "main"
|
|||
optional = false
|
||||
python-versions = ">=3.8.0"
|
||||
files = [
|
||||
{file = "torch-2.0.0-1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:c9090bda7d2eeeecd74f51b721420dbeb44f838d4536cc1b284e879417e3064a"},
|
||||
{file = "torch-2.0.0-1-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:bd42db2a48a20574d2c33489e120e9f32789c4dc13c514b0c44272972d14a2d7"},
|
||||
{file = "torch-2.0.0-1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8969aa8375bcbc0c2993e7ede0a7f889df9515f18b9b548433f412affed478d9"},
|
||||
{file = "torch-2.0.0-1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:ab2da16567cb55b67ae39e32d520d68ec736191d88ac79526ca5874754c32203"},
|
||||
{file = "torch-2.0.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:7a9319a67294ef02459a19738bbfa8727bb5307b822dadd708bc2ccf6c901aca"},
|
||||
{file = "torch-2.0.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9f01fe1f6263f31bd04e1757946fd63ad531ae37f28bb2dbf66f5c826ee089f4"},
|
||||
{file = "torch-2.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:527f4ae68df7b8301ee6b1158ca56350282ea633686537b30dbb5d7b4a52622a"},
|
||||
|
|
@ -4111,6 +4127,15 @@ category = "main"
|
|||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "triton-2.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:38806ee9663f4b0f7cd64790e96c579374089e58f49aac4a6608121aa55e2505"},
|
||||
{file = "triton-2.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:226941c7b8595219ddef59a1fdb821e8c744289a132415ddd584facedeb475b1"},
|
||||
{file = "triton-2.0.0-1-cp36-cp36m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4c9fc8c89874bc48eb7e7b2107a9b8d2c0bf139778637be5bfccb09191685cfd"},
|
||||
{file = "triton-2.0.0-1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d2684b6a60b9f174f447f36f933e9a45f31db96cb723723ecd2dcfd1c57b778b"},
|
||||
{file = "triton-2.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9d4978298b74fcf59a75fe71e535c092b023088933b2f1df933ec32615e4beef"},
|
||||
{file = "triton-2.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:74f118c12b437fb2ca25e1a04759173b517582fcf4c7be11913316c764213656"},
|
||||
{file = "triton-2.0.0-1-pp37-pypy37_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9618815a8da1d9157514f08f855d9e9ff92e329cd81c0305003eb9ec25cc5add"},
|
||||
{file = "triton-2.0.0-1-pp38-pypy38_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aca3303629cd3136375b82cb9921727f804e47ebee27b2677fef23005c3851a"},
|
||||
{file = "triton-2.0.0-1-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3e13aa8b527c9b642e3a9defcc0fbd8ffbe1c80d8ac8c15a01692478dc64d8a"},
|
||||
{file = "triton-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f05a7e64e4ca0565535e3d5d3405d7e49f9d308505bb7773d21fb26a4c008c2"},
|
||||
{file = "triton-2.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb4b99ca3c6844066e516658541d876c28a5f6e3a852286bbc97ad57134827fd"},
|
||||
{file = "triton-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47b4d70dc92fb40af553b4460492c31dc7d3a114a979ffb7a5cdedb7eb546c08"},
|
||||
|
|
@ -4687,4 +4712,4 @@ cffi = ["cffi (>=1.11)"]
|
|||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "4f27ad94f244998e9e79fe1ae733cc786d82025f022c1d337e5c84d0393947b6"
|
||||
content-hash = "313af2197643e0cbac777d28d0a996e1f61f7c1f809a375b5196cc5942c5cc4b"
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ pypdf = "^3.7.1"
|
|||
lxml = "^4.9.2"
|
||||
pysrt = "^1.1.2"
|
||||
fake-useragent = "^1.1.3"
|
||||
docstring-parser = "^0.15"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^23.1.0"
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import inspect
|
|||
import re
|
||||
from typing import Dict, Optional
|
||||
|
||||
from docstring_parser import parse # type: ignore
|
||||
|
||||
from langflow.template.constants import FORCE_SHOW_FIELDS
|
||||
from langflow.utils import constants
|
||||
|
||||
|
|
@ -65,7 +67,8 @@ def build_template_from_function(
|
|||
if v.__annotations__["return"].__name__ == name:
|
||||
_class = v.__annotations__["return"]
|
||||
|
||||
docs = get_class_doc(_class)
|
||||
# Get the docstring
|
||||
docs = parse(_class.__doc__)
|
||||
|
||||
variables = {"_type": _type}
|
||||
for class_field_items, value in _class.__fields__.items():
|
||||
|
|
@ -86,8 +89,8 @@ def build_template_from_function(
|
|||
variables[class_field_items][name_] = value_
|
||||
|
||||
variables[class_field_items]["placeholder"] = (
|
||||
docs["Attributes"][class_field_items]
|
||||
if class_field_items in docs["Attributes"]
|
||||
docs.params[class_field_items]
|
||||
if class_field_items in docs.params
|
||||
else ""
|
||||
)
|
||||
# Adding function to base classes to allow
|
||||
|
|
@ -98,7 +101,7 @@ def build_template_from_function(
|
|||
|
||||
return {
|
||||
"template": format_dict(variables, name),
|
||||
"description": docs["Description"],
|
||||
"description": docs.short_description or "",
|
||||
"base_classes": base_classes,
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +120,7 @@ def build_template_from_class(
|
|||
_class = v
|
||||
|
||||
# Get the docstring
|
||||
docs = get_class_doc(_class)
|
||||
docs = parse(_class.__doc__)
|
||||
|
||||
variables = {"_type": _type}
|
||||
|
||||
|
|
@ -140,8 +143,8 @@ def build_template_from_class(
|
|||
variables[class_field_items][name_] = value_
|
||||
|
||||
variables[class_field_items]["placeholder"] = (
|
||||
docs["Attributes"][class_field_items]
|
||||
if class_field_items in docs["Attributes"]
|
||||
docs.params[class_field_items]
|
||||
if class_field_items in docs.params
|
||||
else ""
|
||||
)
|
||||
base_classes = get_base_classes(_class)
|
||||
|
|
@ -151,7 +154,7 @@ def build_template_from_class(
|
|||
base_classes.append("function")
|
||||
return {
|
||||
"template": format_dict(variables, name),
|
||||
"description": docs["Description"],
|
||||
"description": docs.short_description or "",
|
||||
"base_classes": base_classes,
|
||||
}
|
||||
|
||||
|
|
@ -188,60 +191,6 @@ def get_default_factory(module: str, function: str):
|
|||
return None
|
||||
|
||||
|
||||
def get_class_doc(class_name):
|
||||
"""
|
||||
Extracts information from the docstring of a given class.
|
||||
|
||||
Args:
|
||||
class_name: the class to extract information from
|
||||
|
||||
Returns:
|
||||
A dictionary containing the extracted information, with keys
|
||||
for 'Description', 'Parameters', 'Attributes', and 'Returns'.
|
||||
"""
|
||||
# Template
|
||||
data = {
|
||||
"Description": "",
|
||||
"Parameters": {},
|
||||
"Attributes": {},
|
||||
"Example": [],
|
||||
"Returns": {},
|
||||
}
|
||||
|
||||
# Get the class docstring
|
||||
docstring = class_name.__doc__
|
||||
|
||||
if not docstring:
|
||||
return data
|
||||
|
||||
# Parse the docstring to extract information
|
||||
lines = docstring.split("\n")
|
||||
|
||||
current_section = "Description"
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
|
||||
if not line:
|
||||
continue
|
||||
|
||||
if (
|
||||
line.startswith(tuple(data.keys()))
|
||||
and len(line.split()) == 1
|
||||
and line.endswith(":")
|
||||
):
|
||||
current_section = line[:-1]
|
||||
continue
|
||||
|
||||
if current_section in ["Description", "Example"]:
|
||||
data[current_section] += line
|
||||
else:
|
||||
param, desc = line.split(":")
|
||||
data[current_section][param.strip()] = desc.strip()
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def format_dict(d, name: Optional[str] = None):
|
||||
"""
|
||||
Formats a dictionary by removing certain keys and modifying the
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@ from langflow.utils.util import (
|
|||
build_template_from_function,
|
||||
format_dict,
|
||||
get_base_classes,
|
||||
get_class_doc,
|
||||
get_default_factory,
|
||||
)
|
||||
from pydantic import BaseModel
|
||||
|
|
@ -280,12 +279,3 @@ def test_get_default_factory():
|
|||
default_value = get_default_factory(module_name, function_repr)
|
||||
|
||||
assert default_value == "default_value"
|
||||
|
||||
|
||||
# Test get_class_doc
|
||||
def test_get_class_doc():
|
||||
class_doc_parent = get_class_doc(Parent)
|
||||
class_doc_child = get_class_doc(Child)
|
||||
|
||||
assert class_doc_parent["Description"] == "Parent Class"
|
||||
assert class_doc_child["Description"] == "Child Class"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue