improve: exract Code Node provider for each supported scripting language (#4164)
This commit is contained in:
parent
de3a7603ac
commit
083ef2e6fc
13 changed files with 174 additions and 97 deletions
0
api/core/helper/code_executor/__init__.py
Normal file
0
api/core/helper/code_executor/__init__.py
Normal file
|
|
@ -10,9 +10,10 @@ from yarl import URL
|
|||
|
||||
from config import get_env
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.javascript_transformer import NodeJsTemplateTransformer
|
||||
from core.helper.code_executor.jinja2_transformer import Jinja2TemplateTransformer
|
||||
from core.helper.code_executor.python_transformer import PYTHON_STANDARD_PACKAGES, PythonTemplateTransformer
|
||||
from core.helper.code_executor.javascript.javascript_transformer import NodeJsTemplateTransformer
|
||||
from core.helper.code_executor.jinja2.jinja2_transformer import Jinja2TemplateTransformer
|
||||
from core.helper.code_executor.python3.python3_transformer import PYTHON_STANDARD_PACKAGES, Python3TemplateTransformer
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -34,6 +35,7 @@ class CodeExecutionResponse(BaseModel):
|
|||
message: str
|
||||
data: Data
|
||||
|
||||
|
||||
class CodeLanguage(str, Enum):
|
||||
PYTHON3 = 'python3'
|
||||
JINJA2 = 'jinja2'
|
||||
|
|
@ -44,8 +46,8 @@ class CodeExecutor:
|
|||
dependencies_cache = {}
|
||||
dependencies_cache_lock = Lock()
|
||||
|
||||
code_template_transformers = {
|
||||
CodeLanguage.PYTHON3: PythonTemplateTransformer,
|
||||
code_template_transformers: dict[CodeLanguage, type[TemplateTransformer]] = {
|
||||
CodeLanguage.PYTHON3: Python3TemplateTransformer,
|
||||
CodeLanguage.JINJA2: Jinja2TemplateTransformer,
|
||||
CodeLanguage.JAVASCRIPT: NodeJsTemplateTransformer,
|
||||
}
|
||||
|
|
@ -56,6 +58,10 @@ class CodeExecutor:
|
|||
CodeLanguage.PYTHON3: CodeLanguage.PYTHON3,
|
||||
}
|
||||
|
||||
supported_dependencies_languages: set[CodeLanguage] = {
|
||||
CodeLanguage.PYTHON3
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def execute_code(cls,
|
||||
language: Literal['python3', 'javascript', 'jinja2'],
|
||||
|
|
@ -133,7 +139,10 @@ class CodeExecutor:
|
|||
return template_transformer.transform_response(response)
|
||||
|
||||
@classmethod
|
||||
def list_dependencies(cls, language: Literal['python3']) -> list[CodeDependency]:
|
||||
def list_dependencies(cls, language: str) -> list[CodeDependency]:
|
||||
if language not in cls.supported_dependencies_languages:
|
||||
return []
|
||||
|
||||
with cls.dependencies_cache_lock:
|
||||
if language in cls.dependencies_cache:
|
||||
# check expiration
|
||||
|
|
|
|||
55
api/core/helper/code_executor/code_node_provider.py
Normal file
55
api/core/helper/code_executor/code_node_provider.py
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
from abc import abstractmethod
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from core.helper.code_executor.code_executor import CodeExecutor
|
||||
|
||||
|
||||
class CodeNodeProvider(BaseModel):
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_language() -> str:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def is_accept_language(cls, language: str) -> bool:
|
||||
return language == cls.get_language()
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def get_default_code(cls) -> str:
|
||||
"""
|
||||
get default code in specific programming language for the code node
|
||||
"""
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def get_default_available_packages(cls) -> list[dict]:
|
||||
return [p.dict() for p in CodeExecutor.list_dependencies(cls.get_language())]
|
||||
|
||||
@classmethod
|
||||
def get_default_config(cls) -> dict:
|
||||
return {
|
||||
"type": "code",
|
||||
"config": {
|
||||
"variables": [
|
||||
{
|
||||
"variable": "arg1",
|
||||
"value_selector": []
|
||||
},
|
||||
{
|
||||
"variable": "arg2",
|
||||
"value_selector": []
|
||||
}
|
||||
],
|
||||
"code_language": cls.get_language(),
|
||||
"code": cls.get_default_code(),
|
||||
"outputs": {
|
||||
"result": {
|
||||
"type": "string",
|
||||
"children": None
|
||||
}
|
||||
}
|
||||
},
|
||||
"available_dependencies": cls.get_default_available_packages(),
|
||||
}
|
||||
|
|
@ -3,4 +3,4 @@ from pydantic import BaseModel
|
|||
|
||||
class CodeDependency(BaseModel):
|
||||
name: str
|
||||
version: str
|
||||
version: str
|
||||
|
|
|
|||
0
api/core/helper/code_executor/javascript/__init__.py
Normal file
0
api/core/helper/code_executor/javascript/__init__.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
from textwrap import dedent
|
||||
|
||||
from core.helper.code_executor.code_executor import CodeLanguage
|
||||
from core.helper.code_executor.code_node_provider import CodeNodeProvider
|
||||
|
||||
|
||||
class JavascriptCodeProvider(CodeNodeProvider):
|
||||
@staticmethod
|
||||
def get_language() -> str:
|
||||
return CodeLanguage.JAVASCRIPT
|
||||
|
||||
@classmethod
|
||||
def get_default_code(cls) -> str:
|
||||
return dedent(
|
||||
"""
|
||||
function main({arg1, arg2}) {
|
||||
return {
|
||||
result: arg1 + arg2
|
||||
}
|
||||
}
|
||||
""")
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
import json
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
NODEJS_RUNNER = """// declare main function here
|
||||
{{code}}
|
||||
|
||||
// execute main function, and return the result
|
||||
// inputs is a dict, unstructured inputs
|
||||
output = main({{inputs}})
|
||||
|
||||
// convert output to json and print
|
||||
output = JSON.stringify(output)
|
||||
|
||||
result = `<<RESULT>>${output}<<RESULT>>`
|
||||
|
||||
console.log(result)
|
||||
"""
|
||||
|
||||
NODEJS_PRELOAD = """"""
|
||||
|
||||
|
||||
class NodeJsTemplateTransformer(TemplateTransformer):
|
||||
@classmethod
|
||||
def transform_caller(cls, code: str, inputs: dict,
|
||||
dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
|
||||
"""
|
||||
Transform code to python runner
|
||||
:param code: code
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
|
||||
# transform inputs to json string
|
||||
inputs_str = json.dumps(inputs, indent=4, ensure_ascii=False)
|
||||
|
||||
# replace code and inputs
|
||||
runner = NODEJS_RUNNER.replace('{{code}}', code)
|
||||
runner = runner.replace('{{inputs}}', inputs_str)
|
||||
|
||||
return runner, NODEJS_PRELOAD, []
|
||||
|
||||
@classmethod
|
||||
def transform_response(cls, response: str) -> dict:
|
||||
"""
|
||||
Transform response to dict
|
||||
:param response: response
|
||||
:return:
|
||||
"""
|
||||
# extract result
|
||||
result = re.search(r'<<RESULT>>(.*)<<RESULT>>', response, re.DOTALL)
|
||||
if not result:
|
||||
raise ValueError('Failed to parse result')
|
||||
result = result.group(1)
|
||||
return json.loads(result)
|
||||
0
api/core/helper/code_executor/jinja2/__init__.py
Normal file
0
api/core/helper/code_executor/jinja2/__init__.py
Normal file
17
api/core/helper/code_executor/jinja2/jinja2_formatter.py
Normal file
17
api/core/helper/code_executor/jinja2/jinja2_formatter.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from core.helper.code_executor.code_executor import CodeExecutor, CodeLanguage
|
||||
|
||||
|
||||
class Jinja2Formatter:
|
||||
@classmethod
|
||||
def format(cls, template: str, inputs: str) -> str:
|
||||
"""
|
||||
Format template
|
||||
:param template: template
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
result = CodeExecutor.execute_workflow_code_template(
|
||||
language=CodeLanguage.JINJA2, code=template, inputs=inputs
|
||||
)
|
||||
|
||||
return result['result']
|
||||
108
api/core/helper/code_executor/jinja2/jinja2_transformer.py
Normal file
108
api/core/helper/code_executor/jinja2/jinja2_transformer.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
import json
|
||||
import re
|
||||
from base64 import b64encode
|
||||
from typing import Optional
|
||||
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.python3.python3_transformer import PYTHON_STANDARD_PACKAGES
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
PYTHON_RUNNER = """
|
||||
import jinja2
|
||||
from json import loads
|
||||
from base64 import b64decode
|
||||
|
||||
template = jinja2.Template('''{{code}}''')
|
||||
|
||||
def main(**inputs):
|
||||
return template.render(**inputs)
|
||||
|
||||
# execute main function, and return the result
|
||||
inputs = b64decode('{{inputs}}').decode('utf-8')
|
||||
output = main(**loads(inputs))
|
||||
|
||||
result = f'''<<RESULT>>{output}<<RESULT>>'''
|
||||
|
||||
print(result)
|
||||
|
||||
"""
|
||||
|
||||
JINJA2_PRELOAD_TEMPLATE = """{% set fruits = ['Apple'] %}
|
||||
{{ 'a' }}
|
||||
{% for fruit in fruits %}
|
||||
<li>{{ fruit }}</li>
|
||||
{% endfor %}
|
||||
{% if fruits|length > 1 %}
|
||||
1
|
||||
{% endif %}
|
||||
{% for i in range(5) %}
|
||||
{% if i == 3 %}{{ i }}{% else %}{% endif %}
|
||||
{% endfor %}
|
||||
{% for i in range(3) %}
|
||||
{{ i + 1 }}
|
||||
{% endfor %}
|
||||
{% macro say_hello() %}a{{ 'b' }}{% endmacro %}
|
||||
{{ s }}{{ say_hello() }}"""
|
||||
|
||||
JINJA2_PRELOAD = f"""
|
||||
import jinja2
|
||||
from base64 import b64decode
|
||||
|
||||
def _jinja2_preload_():
|
||||
# prepare jinja2 environment, load template and render before to avoid sandbox issue
|
||||
template = jinja2.Template('''{JINJA2_PRELOAD_TEMPLATE}''')
|
||||
template.render(s='a')
|
||||
|
||||
if __name__ == '__main__':
|
||||
_jinja2_preload_()
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class Jinja2TemplateTransformer(TemplateTransformer):
|
||||
@classmethod
|
||||
def transform_caller(cls, code: str, inputs: dict,
|
||||
dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
|
||||
"""
|
||||
Transform code to python runner
|
||||
:param code: code
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
|
||||
inputs_str = b64encode(json.dumps(inputs, ensure_ascii=False).encode()).decode('utf-8')
|
||||
|
||||
# transform jinja2 template to python code
|
||||
runner = PYTHON_RUNNER.replace('{{code}}', code)
|
||||
runner = runner.replace('{{inputs}}', inputs_str)
|
||||
|
||||
if not dependencies:
|
||||
dependencies = []
|
||||
|
||||
# add native packages and jinja2
|
||||
for package in PYTHON_STANDARD_PACKAGES.union(['jinja2']):
|
||||
dependencies.append(CodeDependency(name=package, version=''))
|
||||
|
||||
# deduplicate
|
||||
dependencies = list({
|
||||
dep.name: dep for dep in dependencies if dep.name
|
||||
}.values())
|
||||
|
||||
return runner, JINJA2_PRELOAD, dependencies
|
||||
|
||||
@classmethod
|
||||
def transform_response(cls, response: str) -> dict:
|
||||
"""
|
||||
Transform response to dict
|
||||
:param response: response
|
||||
:return:
|
||||
"""
|
||||
# extract result
|
||||
result = re.search(r'<<RESULT>>(.*)<<RESULT>>', response, re.DOTALL)
|
||||
if not result:
|
||||
raise ValueError('Failed to parse result')
|
||||
result = result.group(1)
|
||||
|
||||
return {
|
||||
'result': result
|
||||
}
|
||||
0
api/core/helper/code_executor/python3/__init__.py
Normal file
0
api/core/helper/code_executor/python3/__init__.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
from textwrap import dedent
|
||||
|
||||
from core.helper.code_executor.code_executor import CodeLanguage
|
||||
from core.helper.code_executor.code_node_provider import CodeNodeProvider
|
||||
|
||||
|
||||
class Python3CodeProvider(CodeNodeProvider):
|
||||
@staticmethod
|
||||
def get_language() -> str:
|
||||
return CodeLanguage.PYTHON3
|
||||
|
||||
@classmethod
|
||||
def get_default_code(cls) -> str:
|
||||
return dedent(
|
||||
"""
|
||||
def main(arg1: int, arg2: int) -> dict:
|
||||
return {
|
||||
"result": arg1 + arg2,
|
||||
}
|
||||
""")
|
||||
83
api/core/helper/code_executor/python3/python3_transformer.py
Normal file
83
api/core/helper/code_executor/python3/python3_transformer.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import json
|
||||
import re
|
||||
from base64 import b64encode
|
||||
from textwrap import dedent
|
||||
from typing import Optional
|
||||
|
||||
from core.helper.code_executor.entities import CodeDependency
|
||||
from core.helper.code_executor.template_transformer import TemplateTransformer
|
||||
|
||||
PYTHON_RUNNER = dedent("""
|
||||
# declare main function here
|
||||
{{code}}
|
||||
|
||||
from json import loads, dumps
|
||||
from base64 import b64decode
|
||||
|
||||
# execute main function, and return the result
|
||||
# inputs is a dict, and it
|
||||
inputs = b64decode('{{inputs}}').decode('utf-8')
|
||||
output = main(**json.loads(inputs))
|
||||
|
||||
# convert output to json and print
|
||||
output = dumps(output, indent=4)
|
||||
|
||||
result = f'''<<RESULT>>
|
||||
{output}
|
||||
<<RESULT>>'''
|
||||
|
||||
print(result)
|
||||
""")
|
||||
|
||||
PYTHON_PRELOAD = """"""
|
||||
|
||||
PYTHON_STANDARD_PACKAGES = {
|
||||
'json', 'datetime', 'math', 'random', 're', 'string', 'sys', 'time', 'traceback', 'uuid', 'os', 'base64',
|
||||
'hashlib', 'hmac', 'binascii', 'collections', 'functools', 'operator', 'itertools', 'uuid',
|
||||
}
|
||||
|
||||
|
||||
class Python3TemplateTransformer(TemplateTransformer):
|
||||
@classmethod
|
||||
def transform_caller(cls, code: str, inputs: dict,
|
||||
dependencies: Optional[list[CodeDependency]] = None) -> tuple[str, str, list[CodeDependency]]:
|
||||
"""
|
||||
Transform code to python runner
|
||||
:param code: code
|
||||
:param inputs: inputs
|
||||
:return:
|
||||
"""
|
||||
|
||||
# transform inputs to json string
|
||||
inputs_str = b64encode(json.dumps(inputs, ensure_ascii=False).encode()).decode('utf-8')
|
||||
|
||||
# replace code and inputs
|
||||
runner = PYTHON_RUNNER.replace('{{code}}', code)
|
||||
runner = runner.replace('{{inputs}}', inputs_str)
|
||||
|
||||
# add standard packages
|
||||
if dependencies is None:
|
||||
dependencies = []
|
||||
|
||||
for package in PYTHON_STANDARD_PACKAGES:
|
||||
if package not in dependencies:
|
||||
dependencies.append(CodeDependency(name=package, version=''))
|
||||
|
||||
# deduplicate
|
||||
dependencies = list({dep.name: dep for dep in dependencies if dep.name}.values())
|
||||
|
||||
return runner, PYTHON_PRELOAD, dependencies
|
||||
|
||||
@classmethod
|
||||
def transform_response(cls, response: str) -> dict:
|
||||
"""
|
||||
Transform response to dict
|
||||
:param response: response
|
||||
:return:
|
||||
"""
|
||||
# extract result
|
||||
result = re.search(r'<<RESULT>>(.*?)<<RESULT>>', response, re.DOTALL)
|
||||
if not result:
|
||||
raise ValueError('Failed to parse result')
|
||||
result = result.group(1)
|
||||
return json.loads(result)
|
||||
Loading…
Reference in a new issue