[Chore/Refactor] Switch from MyPy to Basedpyright for type checking (#25047)
Signed-off-by: -LAN- <laipz8200@outlook.com>
This commit is contained in:
parent
1fff4620e6
commit
9d5956cef8
84 changed files with 2380 additions and 2351 deletions
10
.github/workflows/api-tests.yml
vendored
10
.github/workflows/api-tests.yml
vendored
|
|
@ -62,14 +62,8 @@ jobs:
|
||||||
- name: Run dify config tests
|
- name: Run dify config tests
|
||||||
run: uv run --project api dev/pytest/pytest_config_tests.py
|
run: uv run --project api dev/pytest/pytest_config_tests.py
|
||||||
|
|
||||||
- name: MyPy Cache
|
- name: Run Basedpyright Checks
|
||||||
uses: actions/cache@v4
|
run: dev/basedpyright-check
|
||||||
with:
|
|
||||||
path: api/.mypy_cache
|
|
||||||
key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }}
|
|
||||||
|
|
||||||
- name: Run MyPy Checks
|
|
||||||
run: dev/mypy-check
|
|
||||||
|
|
||||||
- name: Set up dotenvs
|
- name: Set up dotenvs
|
||||||
run: |
|
run: |
|
||||||
|
|
|
||||||
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -123,7 +123,7 @@ venv.bak/
|
||||||
# mkdocs documentation
|
# mkdocs documentation
|
||||||
/site
|
/site
|
||||||
|
|
||||||
# mypy
|
# type checking
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.dmypy.json
|
.dmypy.json
|
||||||
dmypy.json
|
dmypy.json
|
||||||
|
|
@ -195,7 +195,6 @@ sdks/python-client/dify_client.egg-info
|
||||||
.vscode/*
|
.vscode/*
|
||||||
!.vscode/launch.json.template
|
!.vscode/launch.json.template
|
||||||
!.vscode/README.md
|
!.vscode/README.md
|
||||||
pyrightconfig.json
|
|
||||||
api/.vscode
|
api/.vscode
|
||||||
# vscode Code History Extension
|
# vscode Code History Extension
|
||||||
.history
|
.history
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,7 @@ uv run --project api pytest tests/integration_tests/ # Integration tests
|
||||||
./dev/reformat # Run all formatters and linters
|
./dev/reformat # Run all formatters and linters
|
||||||
uv run --project api ruff check --fix ./ # Fix linting issues
|
uv run --project api ruff check --fix ./ # Fix linting issues
|
||||||
uv run --project api ruff format ./ # Format code
|
uv run --project api ruff format ./ # Format code
|
||||||
uv run --project api mypy . # Type checking
|
uv run --directory api basedpyright # Type checking
|
||||||
```
|
```
|
||||||
|
|
||||||
### Frontend (Web)
|
### Frontend (Web)
|
||||||
|
|
|
||||||
|
|
@ -108,5 +108,5 @@ uv run celery -A app.celery beat
|
||||||
../dev/reformat # Run all formatters and linters
|
../dev/reformat # Run all formatters and linters
|
||||||
uv run ruff check --fix ./ # Fix linting issues
|
uv run ruff check --fix ./ # Fix linting issues
|
||||||
uv run ruff format ./ # Format code
|
uv run ruff format ./ # Format code
|
||||||
uv run mypy . # Type checking
|
uv run basedpyright . # Type checking
|
||||||
```
|
```
|
||||||
|
|
|
||||||
|
|
@ -571,7 +571,7 @@ def old_metadata_migration():
|
||||||
for document in documents:
|
for document in documents:
|
||||||
if document.doc_metadata:
|
if document.doc_metadata:
|
||||||
doc_metadata = document.doc_metadata
|
doc_metadata = document.doc_metadata
|
||||||
for key, value in doc_metadata.items():
|
for key in doc_metadata:
|
||||||
for field in BuiltInField:
|
for field in BuiltInField:
|
||||||
if field.value == key:
|
if field.value == key:
|
||||||
break
|
break
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ class NacosSettingsSource(RemoteSettingsSource):
|
||||||
try:
|
try:
|
||||||
content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
|
content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
|
||||||
self.remote_configs = self._parse_config(content)
|
self.remote_configs = self._parse_config(content)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("[get-access-token] exception occurred")
|
logger.exception("[get-access-token] exception occurred")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,6 +77,6 @@ class NacosHttpClient:
|
||||||
self.token = response_data.get("accessToken")
|
self.token = response_data.get("accessToken")
|
||||||
self.token_ttl = response_data.get("tokenTtl", 18000)
|
self.token_ttl = response_data.get("tokenTtl", 18000)
|
||||||
self.token_expire_time = current_time + self.token_ttl - 10
|
self.token_expire_time = current_time + self.token_ttl - 10
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("[get-access-token] exception occur")
|
logger.exception("[get-access-token] exception occur")
|
||||||
raise
|
raise
|
||||||
|
|
|
||||||
|
|
@ -130,7 +130,7 @@ class ResetPasswordSendEmailApi(Resource):
|
||||||
language = "en-US"
|
language = "en-US"
|
||||||
try:
|
try:
|
||||||
account = AccountService.get_user_through_email(args["email"])
|
account = AccountService.get_user_through_email(args["email"])
|
||||||
except AccountRegisterError as are:
|
except AccountRegisterError:
|
||||||
raise AccountInFreezeError()
|
raise AccountInFreezeError()
|
||||||
|
|
||||||
if account is None:
|
if account is None:
|
||||||
|
|
@ -162,7 +162,7 @@ class EmailCodeLoginSendEmailApi(Resource):
|
||||||
language = "en-US"
|
language = "en-US"
|
||||||
try:
|
try:
|
||||||
account = AccountService.get_user_through_email(args["email"])
|
account = AccountService.get_user_through_email(args["email"])
|
||||||
except AccountRegisterError as are:
|
except AccountRegisterError:
|
||||||
raise AccountInFreezeError()
|
raise AccountInFreezeError()
|
||||||
|
|
||||||
if account is None:
|
if account is None:
|
||||||
|
|
@ -200,7 +200,7 @@ class EmailCodeLoginApi(Resource):
|
||||||
AccountService.revoke_email_code_login_token(args["token"])
|
AccountService.revoke_email_code_login_token(args["token"])
|
||||||
try:
|
try:
|
||||||
account = AccountService.get_user_through_email(user_email)
|
account = AccountService.get_user_through_email(user_email)
|
||||||
except AccountRegisterError as are:
|
except AccountRegisterError:
|
||||||
raise AccountInFreezeError()
|
raise AccountInFreezeError()
|
||||||
if account:
|
if account:
|
||||||
tenants = TenantService.get_join_tenants(account)
|
tenants = TenantService.get_join_tenants(account)
|
||||||
|
|
@ -223,7 +223,7 @@ class EmailCodeLoginApi(Resource):
|
||||||
)
|
)
|
||||||
except WorkSpaceNotAllowedCreateError:
|
except WorkSpaceNotAllowedCreateError:
|
||||||
raise NotAllowedCreateWorkspace()
|
raise NotAllowedCreateWorkspace()
|
||||||
except AccountRegisterError as are:
|
except AccountRegisterError:
|
||||||
raise AccountInFreezeError()
|
raise AccountInFreezeError()
|
||||||
except WorkspacesLimitExceededError:
|
except WorkspacesLimitExceededError:
|
||||||
raise WorkspacesLimitExceeded()
|
raise WorkspacesLimitExceeded()
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,7 @@ class FilePreviewApi(Resource):
|
||||||
args = file_preview_parser.parse_args()
|
args = file_preview_parser.parse_args()
|
||||||
|
|
||||||
# Validate file ownership and get file objects
|
# Validate file ownership and get file objects
|
||||||
message_file, upload_file = self._validate_file_ownership(file_id, app_model.id)
|
_, upload_file = self._validate_file_ownership(file_id, app_model.id)
|
||||||
|
|
||||||
# Get file content generator
|
# Get file content generator
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -410,7 +410,7 @@ class DocumentUpdateByFileApi(DatasetApiResource):
|
||||||
DocumentService.document_create_args_validate(knowledge_config)
|
DocumentService.document_create_args_validate(knowledge_config)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
documents, batch = DocumentService.save_document_with_dataset_id(
|
documents, _ = DocumentService.save_document_with_dataset_id(
|
||||||
dataset=dataset,
|
dataset=dataset,
|
||||||
knowledge_config=knowledge_config,
|
knowledge_config=knowledge_config,
|
||||||
account=dataset.created_by_account,
|
account=dataset.created_by_account,
|
||||||
|
|
|
||||||
|
|
@ -26,7 +26,7 @@ class MoreLikeThisConfigManager:
|
||||||
def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
|
def validate_and_set_defaults(cls, config: dict) -> tuple[dict, list[str]]:
|
||||||
try:
|
try:
|
||||||
return AppConfigModel.model_validate(config).model_dump(), ["more_like_this"]
|
return AppConfigModel.model_validate(config).model_dump(), ["more_like_this"]
|
||||||
except ValidationError as e:
|
except ValidationError:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"more_like_this must be of dict type and enabled in more_like_this must be of boolean type"
|
"more_like_this must be of dict type and enabled in more_like_this must be of boolean type"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -310,13 +310,8 @@ class AdvancedChatAppGenerateTaskPipeline:
|
||||||
err = self._base_task_pipeline._handle_error(event=event, session=session, message_id=self._message_id)
|
err = self._base_task_pipeline._handle_error(event=event, session=session, message_id=self._message_id)
|
||||||
yield self._base_task_pipeline._error_to_stream_response(err)
|
yield self._base_task_pipeline._error_to_stream_response(err)
|
||||||
|
|
||||||
def _handle_workflow_started_event(
|
def _handle_workflow_started_event(self, **kwargs) -> Generator[StreamResponse, None, None]:
|
||||||
self, event: QueueWorkflowStartedEvent, *, graph_runtime_state: Optional[GraphRuntimeState] = None, **kwargs
|
|
||||||
) -> Generator[StreamResponse, None, None]:
|
|
||||||
"""Handle workflow started events."""
|
"""Handle workflow started events."""
|
||||||
# Override graph runtime state - this is a side effect but necessary
|
|
||||||
graph_runtime_state = event.graph_runtime_state
|
|
||||||
|
|
||||||
with self._database_session() as session:
|
with self._database_session() as session:
|
||||||
workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start()
|
workflow_execution = self._workflow_cycle_manager.handle_workflow_run_start()
|
||||||
self._workflow_run_id = workflow_execution.id_
|
self._workflow_run_id = workflow_execution.id_
|
||||||
|
|
@ -337,15 +332,14 @@ class AdvancedChatAppGenerateTaskPipeline:
|
||||||
"""Handle node retry events."""
|
"""Handle node retry events."""
|
||||||
self._ensure_workflow_initialized()
|
self._ensure_workflow_initialized()
|
||||||
|
|
||||||
with self._database_session() as session:
|
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
|
||||||
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
|
workflow_execution_id=self._workflow_run_id, event=event
|
||||||
workflow_execution_id=self._workflow_run_id, event=event
|
)
|
||||||
)
|
node_retry_resp = self._workflow_response_converter.workflow_node_retry_to_stream_response(
|
||||||
node_retry_resp = self._workflow_response_converter.workflow_node_retry_to_stream_response(
|
event=event,
|
||||||
event=event,
|
task_id=self._application_generate_entity.task_id,
|
||||||
task_id=self._application_generate_entity.task_id,
|
workflow_node_execution=workflow_node_execution,
|
||||||
workflow_node_execution=workflow_node_execution,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
if node_retry_resp:
|
if node_retry_resp:
|
||||||
yield node_retry_resp
|
yield node_retry_resp
|
||||||
|
|
@ -379,13 +373,12 @@ class AdvancedChatAppGenerateTaskPipeline:
|
||||||
self._workflow_response_converter.fetch_files_from_node_outputs(event.outputs or {})
|
self._workflow_response_converter.fetch_files_from_node_outputs(event.outputs or {})
|
||||||
)
|
)
|
||||||
|
|
||||||
with self._database_session() as session:
|
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_success(event=event)
|
||||||
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_success(event=event)
|
node_finish_resp = self._workflow_response_converter.workflow_node_finish_to_stream_response(
|
||||||
node_finish_resp = self._workflow_response_converter.workflow_node_finish_to_stream_response(
|
event=event,
|
||||||
event=event,
|
task_id=self._application_generate_entity.task_id,
|
||||||
task_id=self._application_generate_entity.task_id,
|
workflow_node_execution=workflow_node_execution,
|
||||||
workflow_node_execution=workflow_node_execution,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
self._save_output_for_event(event, workflow_node_execution.id)
|
self._save_output_for_event(event, workflow_node_execution.id)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -159,7 +159,7 @@ class AppQueueManager:
|
||||||
def _check_for_sqlalchemy_models(self, data: Any):
|
def _check_for_sqlalchemy_models(self, data: Any):
|
||||||
# from entity to dict or list
|
# from entity to dict or list
|
||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
for key, value in data.items():
|
for value in data.values():
|
||||||
self._check_for_sqlalchemy_models(value)
|
self._check_for_sqlalchemy_models(value)
|
||||||
elif isinstance(data, list):
|
elif isinstance(data, list):
|
||||||
for item in data:
|
for item in data:
|
||||||
|
|
|
||||||
|
|
@ -300,16 +300,15 @@ class WorkflowAppGenerateTaskPipeline:
|
||||||
"""Handle node retry events."""
|
"""Handle node retry events."""
|
||||||
self._ensure_workflow_initialized()
|
self._ensure_workflow_initialized()
|
||||||
|
|
||||||
with self._database_session() as session:
|
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
|
||||||
workflow_node_execution = self._workflow_cycle_manager.handle_workflow_node_execution_retried(
|
workflow_execution_id=self._workflow_run_id,
|
||||||
workflow_execution_id=self._workflow_run_id,
|
event=event,
|
||||||
event=event,
|
)
|
||||||
)
|
response = self._workflow_response_converter.workflow_node_retry_to_stream_response(
|
||||||
response = self._workflow_response_converter.workflow_node_retry_to_stream_response(
|
event=event,
|
||||||
event=event,
|
task_id=self._application_generate_entity.task_id,
|
||||||
task_id=self._application_generate_entity.task_id,
|
workflow_node_execution=workflow_node_execution,
|
||||||
workflow_node_execution=workflow_node_execution,
|
)
|
||||||
)
|
|
||||||
|
|
||||||
if response:
|
if response:
|
||||||
yield response
|
yield response
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from enum import Enum
|
from enum import StrEnum
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
|
from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
|
||||||
|
|
@ -11,7 +11,7 @@ from core.file import File, FileUploadConfig
|
||||||
from core.model_runtime.entities.model_entities import AIModelEntity
|
from core.model_runtime.entities.model_entities import AIModelEntity
|
||||||
|
|
||||||
|
|
||||||
class InvokeFrom(Enum):
|
class InvokeFrom(StrEnum):
|
||||||
"""
|
"""
|
||||||
Invoke From.
|
Invoke From.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,7 @@ class MessageCycleManager:
|
||||||
try:
|
try:
|
||||||
name = LLMGenerator.generate_conversation_name(app_model.tenant_id, query)
|
name = LLMGenerator.generate_conversation_name(app_model.tenant_id, query)
|
||||||
conversation.name = name
|
conversation.name = name
|
||||||
except Exception as e:
|
except Exception:
|
||||||
if dify_config.DEBUG:
|
if dify_config.DEBUG:
|
||||||
logger.exception("generate conversation name failed, conversation_id: %s", conversation_id)
|
logger.exception("generate conversation name failed, conversation_id: %s", conversation_id)
|
||||||
pass
|
pass
|
||||||
|
|
|
||||||
|
|
@ -67,7 +67,7 @@ class DatasetIndexToolCallbackHandler:
|
||||||
)
|
)
|
||||||
child_chunk = db.session.scalar(child_chunk_stmt)
|
child_chunk = db.session.scalar(child_chunk_stmt)
|
||||||
if child_chunk:
|
if child_chunk:
|
||||||
segment = (
|
_ = (
|
||||||
db.session.query(DocumentSegment)
|
db.session.query(DocumentSegment)
|
||||||
.where(DocumentSegment.id == child_chunk.segment_id)
|
.where(DocumentSegment.id == child_chunk.segment_id)
|
||||||
.update(
|
.update(
|
||||||
|
|
|
||||||
|
|
@ -91,7 +91,7 @@ class Extensible:
|
||||||
|
|
||||||
# Find extension class
|
# Find extension class
|
||||||
extension_class = None
|
extension_class = None
|
||||||
for name, obj in vars(mod).items():
|
for obj in vars(mod).values():
|
||||||
if isinstance(obj, type) and issubclass(obj, cls) and obj != cls:
|
if isinstance(obj, type) and issubclass(obj, cls) and obj != cls:
|
||||||
extension_class = obj
|
extension_class = obj
|
||||||
break
|
break
|
||||||
|
|
@ -123,7 +123,7 @@ class Extensible:
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Error scanning extensions")
|
logger.exception("Error scanning extensions")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -41,9 +41,3 @@ class Extension:
|
||||||
assert module_extension.extension_class is not None
|
assert module_extension.extension_class is not None
|
||||||
t: type = module_extension.extension_class
|
t: type = module_extension.extension_class
|
||||||
return t
|
return t
|
||||||
|
|
||||||
def validate_form_schema(self, module: ExtensionModule, extension_name: str, config: dict) -> None:
|
|
||||||
module_extension = self.module_extension(module, extension_name)
|
|
||||||
form_schema = module_extension.form_schema
|
|
||||||
|
|
||||||
# TODO validate form_schema
|
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,6 @@ class ExternalDataToolFactory:
|
||||||
:param config: the form config data
|
:param config: the form config data
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
code_based_extension.validate_form_schema(ExtensionModule.EXTERNAL_DATA_TOOL, name, config)
|
|
||||||
extension_class = code_based_extension.extension_class(ExtensionModule.EXTERNAL_DATA_TOOL, name)
|
extension_class = code_based_extension.extension_class(ExtensionModule.EXTERNAL_DATA_TOOL, name)
|
||||||
# FIXME mypy issue here, figure out how to fix it
|
# FIXME mypy issue here, figure out how to fix it
|
||||||
extension_class.validate_config(tenant_id, config) # type: ignore
|
extension_class.validate_config(tenant_id, config) # type: ignore
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@ def batch_fetch_plugin_manifests_ignore_deserialization_error(
|
||||||
for plugin in response.json()["data"]["plugins"]:
|
for plugin in response.json()["data"]["plugins"]:
|
||||||
try:
|
try:
|
||||||
result.append(MarketplacePluginDeclaration(**plugin))
|
result.append(MarketplacePluginDeclaration(**plugin))
|
||||||
except Exception as e:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import re
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any, Optional, cast
|
from typing import Any, Optional
|
||||||
|
|
||||||
from flask import current_app
|
from flask import current_app
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
@ -397,7 +397,6 @@ class IndexingRunner:
|
||||||
)
|
)
|
||||||
|
|
||||||
# replace doc id to document model id
|
# replace doc id to document model id
|
||||||
text_docs = cast(list[Document], text_docs)
|
|
||||||
for text_doc in text_docs:
|
for text_doc in text_docs:
|
||||||
if text_doc.metadata is not None:
|
if text_doc.metadata is not None:
|
||||||
text_doc.metadata["document_id"] = dataset_document.id
|
text_doc.metadata["document_id"] = dataset_document.id
|
||||||
|
|
|
||||||
|
|
@ -66,7 +66,7 @@ class LLMGenerator:
|
||||||
try:
|
try:
|
||||||
result_dict = json.loads(cleaned_answer)
|
result_dict = json.loads(cleaned_answer)
|
||||||
answer = result_dict["Your Output"]
|
answer = result_dict["Your Output"]
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError:
|
||||||
logger.exception("Failed to generate name after answer, use query instead")
|
logger.exception("Failed to generate name after answer, use query instead")
|
||||||
answer = query
|
answer = query
|
||||||
name = answer.strip()
|
name = answer.strip()
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,7 @@ def handle_callback(state_key: str, authorization_code: str) -> OAuthCallbackSta
|
||||||
|
|
||||||
def check_support_resource_discovery(server_url: str) -> tuple[bool, str]:
|
def check_support_resource_discovery(server_url: str) -> tuple[bool, str]:
|
||||||
"""Check if the server supports OAuth 2.0 Resource Discovery."""
|
"""Check if the server supports OAuth 2.0 Resource Discovery."""
|
||||||
b_scheme, b_netloc, b_path, b_params, b_query, b_fragment = urlparse(server_url, "", True)
|
b_scheme, b_netloc, b_path, _, b_query, b_fragment = urlparse(server_url, "", True)
|
||||||
url_for_resource_discovery = f"{b_scheme}://{b_netloc}/.well-known/oauth-protected-resource{b_path}"
|
url_for_resource_discovery = f"{b_scheme}://{b_netloc}/.well-known/oauth-protected-resource{b_path}"
|
||||||
if b_query:
|
if b_query:
|
||||||
url_for_resource_discovery += f"?{b_query}"
|
url_for_resource_discovery += f"?{b_query}"
|
||||||
|
|
@ -117,7 +117,7 @@ def check_support_resource_discovery(server_url: str) -> tuple[bool, str]:
|
||||||
else:
|
else:
|
||||||
return False, ""
|
return False, ""
|
||||||
return False, ""
|
return False, ""
|
||||||
except httpx.RequestError as e:
|
except httpx.RequestError:
|
||||||
# Not support resource discovery, fall back to well-known OAuth metadata
|
# Not support resource discovery, fall back to well-known OAuth metadata
|
||||||
return False, ""
|
return False, ""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ import logging
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from contextlib import AbstractContextManager, ExitStack
|
from contextlib import AbstractContextManager, ExitStack
|
||||||
from types import TracebackType
|
from types import TracebackType
|
||||||
from typing import Any, Optional, cast
|
from typing import Any, Optional
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from core.mcp.client.sse_client import sse_client
|
from core.mcp.client.sse_client import sse_client
|
||||||
|
|
@ -116,8 +116,7 @@ class MCPClient:
|
||||||
|
|
||||||
self._session_context = ClientSession(*streams)
|
self._session_context = ClientSession(*streams)
|
||||||
self._session = self._exit_stack.enter_context(self._session_context)
|
self._session = self._exit_stack.enter_context(self._session_context)
|
||||||
session = cast(ClientSession, self._session)
|
self._session.initialize()
|
||||||
session.initialize()
|
|
||||||
return
|
return
|
||||||
|
|
||||||
except MCPAuthError:
|
except MCPAuthError:
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,6 @@ class ModerationFactory:
|
||||||
:param config: the form config data
|
:param config: the form config data
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
code_based_extension.validate_form_schema(ExtensionModule.MODERATION, name, config)
|
|
||||||
extension_class = code_based_extension.extension_class(ExtensionModule.MODERATION, name)
|
extension_class = code_based_extension.extension_class(ExtensionModule.MODERATION, name)
|
||||||
# FIXME: mypy error, try to fix it instead of using type: ignore
|
# FIXME: mypy error, try to fix it instead of using type: ignore
|
||||||
extension_class.validate_config(tenant_id, config) # type: ignore
|
extension_class.validate_config(tenant_id, config) # type: ignore
|
||||||
|
|
|
||||||
|
|
@ -135,7 +135,7 @@ class OutputModeration(BaseModel):
|
||||||
|
|
||||||
result: ModerationOutputsResult = moderation_factory.moderation_for_outputs(moderation_buffer)
|
result: ModerationOutputsResult = moderation_factory.moderation_for_outputs(moderation_buffer)
|
||||||
return result
|
return result
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Moderation Output error, app_id: %s", app_id)
|
logger.exception("Moderation Output error, app_id: %s", app_id)
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -849,7 +849,7 @@ class TraceQueueManager:
|
||||||
if self.trace_instance:
|
if self.trace_instance:
|
||||||
trace_task.app_id = self.app_id
|
trace_task.app_id = self.app_id
|
||||||
trace_manager_queue.put(trace_task)
|
trace_manager_queue.put(trace_task)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Error adding trace task, trace_type %s", trace_task.trace_type)
|
logger.exception("Error adding trace task, trace_type %s", trace_task.trace_type)
|
||||||
finally:
|
finally:
|
||||||
self.start_timer()
|
self.start_timer()
|
||||||
|
|
@ -868,7 +868,7 @@ class TraceQueueManager:
|
||||||
tasks = self.collect_tasks()
|
tasks = self.collect_tasks()
|
||||||
if tasks:
|
if tasks:
|
||||||
self.send_to_celery(tasks)
|
self.send_to_celery(tasks)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Error processing trace tasks")
|
logger.exception("Error processing trace tasks")
|
||||||
|
|
||||||
def start_timer(self):
|
def start_timer(self):
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TypeVar, Union, cast
|
from typing import TypeVar, Union
|
||||||
|
|
||||||
from core.agent.entities import AgentInvokeMessage
|
from core.agent.entities import AgentInvokeMessage
|
||||||
from core.tools.entities.tool_entities import ToolInvokeMessage
|
from core.tools.entities.tool_entities import ToolInvokeMessage
|
||||||
|
|
@ -85,7 +85,7 @@ def merge_blob_chunks(
|
||||||
message=ToolInvokeMessage.BlobMessage(blob=files[chunk_id].data[: files[chunk_id].bytes_written]),
|
message=ToolInvokeMessage.BlobMessage(blob=files[chunk_id].data[: files[chunk_id].bytes_written]),
|
||||||
meta=resp.meta,
|
meta=resp.meta,
|
||||||
)
|
)
|
||||||
yield cast(MessageType, merged_message)
|
yield merged_message
|
||||||
# Clean up the buffer
|
# Clean up the buffer
|
||||||
del files[chunk_id]
|
del files[chunk_id]
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -228,7 +228,7 @@ class AnalyticdbVectorBySql:
|
||||||
)
|
)
|
||||||
documents = []
|
documents = []
|
||||||
for record in cur:
|
for record in cur:
|
||||||
id, vector, score, page_content, metadata = record
|
_, vector, score, page_content, metadata = record
|
||||||
if score >= score_threshold:
|
if score >= score_threshold:
|
||||||
metadata["score"] = score
|
metadata["score"] = score
|
||||||
doc = Document(
|
doc = Document(
|
||||||
|
|
@ -260,7 +260,7 @@ class AnalyticdbVectorBySql:
|
||||||
)
|
)
|
||||||
documents = []
|
documents = []
|
||||||
for record in cur:
|
for record in cur:
|
||||||
id, vector, page_content, metadata, score = record
|
_, vector, page_content, metadata, score = record
|
||||||
metadata["score"] = score
|
metadata["score"] = score
|
||||||
doc = Document(
|
doc = Document(
|
||||||
page_content=page_content,
|
page_content=page_content,
|
||||||
|
|
|
||||||
|
|
@ -701,7 +701,7 @@ class ClickzettaVector(BaseVector):
|
||||||
len(data_rows),
|
len(data_rows),
|
||||||
vector_dimension,
|
vector_dimension,
|
||||||
)
|
)
|
||||||
except (RuntimeError, ValueError, TypeError, ConnectionError) as e:
|
except (RuntimeError, ValueError, TypeError, ConnectionError):
|
||||||
logger.exception("Parameterized SQL execution failed for %d documents", len(data_rows))
|
logger.exception("Parameterized SQL execution failed for %d documents", len(data_rows))
|
||||||
logger.exception("SQL template: %s", insert_sql)
|
logger.exception("SQL template: %s", insert_sql)
|
||||||
logger.exception("Sample data row: %s", data_rows[0] if data_rows else "None")
|
logger.exception("Sample data row: %s", data_rows[0] if data_rows else "None")
|
||||||
|
|
@ -787,7 +787,7 @@ class ClickzettaVector(BaseVector):
|
||||||
document_ids_filter = kwargs.get("document_ids_filter")
|
document_ids_filter = kwargs.get("document_ids_filter")
|
||||||
|
|
||||||
# Handle filter parameter from canvas (workflow)
|
# Handle filter parameter from canvas (workflow)
|
||||||
filter_param = kwargs.get("filter", {})
|
_ = kwargs.get("filter", {})
|
||||||
|
|
||||||
# Build filter clause
|
# Build filter clause
|
||||||
filter_clauses = []
|
filter_clauses = []
|
||||||
|
|
@ -879,7 +879,7 @@ class ClickzettaVector(BaseVector):
|
||||||
document_ids_filter = kwargs.get("document_ids_filter")
|
document_ids_filter = kwargs.get("document_ids_filter")
|
||||||
|
|
||||||
# Handle filter parameter from canvas (workflow)
|
# Handle filter parameter from canvas (workflow)
|
||||||
filter_param = kwargs.get("filter", {})
|
_ = kwargs.get("filter", {})
|
||||||
|
|
||||||
# Build filter clause
|
# Build filter clause
|
||||||
filter_clauses = []
|
filter_clauses = []
|
||||||
|
|
@ -938,7 +938,7 @@ class ClickzettaVector(BaseVector):
|
||||||
metadata = {}
|
metadata = {}
|
||||||
else:
|
else:
|
||||||
metadata = {}
|
metadata = {}
|
||||||
except (json.JSONDecodeError, TypeError) as e:
|
except (json.JSONDecodeError, TypeError):
|
||||||
logger.exception("JSON parsing failed")
|
logger.exception("JSON parsing failed")
|
||||||
# Fallback: extract document_id with regex
|
# Fallback: extract document_id with regex
|
||||||
|
|
||||||
|
|
@ -956,7 +956,7 @@ class ClickzettaVector(BaseVector):
|
||||||
metadata["score"] = 1.0 # Clickzetta doesn't provide relevance scores
|
metadata["score"] = 1.0 # Clickzetta doesn't provide relevance scores
|
||||||
doc = Document(page_content=row[1], metadata=metadata)
|
doc = Document(page_content=row[1], metadata=metadata)
|
||||||
documents.append(doc)
|
documents.append(doc)
|
||||||
except (RuntimeError, ValueError, TypeError, ConnectionError) as e:
|
except (RuntimeError, ValueError, TypeError, ConnectionError):
|
||||||
logger.exception("Full-text search failed")
|
logger.exception("Full-text search failed")
|
||||||
# Fallback to LIKE search if full-text search fails
|
# Fallback to LIKE search if full-text search fails
|
||||||
return self._search_by_like(query, **kwargs)
|
return self._search_by_like(query, **kwargs)
|
||||||
|
|
@ -978,7 +978,7 @@ class ClickzettaVector(BaseVector):
|
||||||
document_ids_filter = kwargs.get("document_ids_filter")
|
document_ids_filter = kwargs.get("document_ids_filter")
|
||||||
|
|
||||||
# Handle filter parameter from canvas (workflow)
|
# Handle filter parameter from canvas (workflow)
|
||||||
filter_param = kwargs.get("filter", {})
|
_ = kwargs.get("filter", {})
|
||||||
|
|
||||||
# Build filter clause
|
# Build filter clause
|
||||||
filter_clauses = []
|
filter_clauses = []
|
||||||
|
|
|
||||||
|
|
@ -212,10 +212,10 @@ class CouchbaseVector(BaseVector):
|
||||||
|
|
||||||
documents_to_insert = [
|
documents_to_insert = [
|
||||||
{"text": text, "embedding": vector, "metadata": metadata}
|
{"text": text, "embedding": vector, "metadata": metadata}
|
||||||
for id, text, vector, metadata in zip(uuids, texts, embeddings, metadatas)
|
for _, text, vector, metadata in zip(uuids, texts, embeddings, metadatas)
|
||||||
]
|
]
|
||||||
for doc, id in zip(documents_to_insert, uuids):
|
for doc, id in zip(documents_to_insert, uuids):
|
||||||
result = self._scope.collection(self._collection_name).upsert(id, doc)
|
_ = self._scope.collection(self._collection_name).upsert(id, doc)
|
||||||
|
|
||||||
doc_ids.extend(uuids)
|
doc_ids.extend(uuids)
|
||||||
|
|
||||||
|
|
@ -241,7 +241,7 @@ class CouchbaseVector(BaseVector):
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self._cluster.query(query, named_parameters={"doc_ids": ids}).execute()
|
self._cluster.query(query, named_parameters={"doc_ids": ids}).execute()
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to delete documents, ids: %s", ids)
|
logger.exception("Failed to delete documents, ids: %s", ids)
|
||||||
|
|
||||||
def delete_by_document_id(self, document_id: str):
|
def delete_by_document_id(self, document_id: str):
|
||||||
|
|
|
||||||
|
|
@ -99,7 +99,7 @@ class MatrixoneVector(BaseVector):
|
||||||
return client
|
return client
|
||||||
try:
|
try:
|
||||||
client.create_full_text_index()
|
client.create_full_text_index()
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to create full text index")
|
logger.exception("Failed to create full text index")
|
||||||
redis_client.set(collection_exist_cache_key, 1, ex=3600)
|
redis_client.set(collection_exist_cache_key, 1, ex=3600)
|
||||||
return client
|
return client
|
||||||
|
|
|
||||||
|
|
@ -197,7 +197,7 @@ class OpenSearchVector(BaseVector):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = self._client.search(index=self._collection_name.lower(), body=query)
|
response = self._client.search(index=self._collection_name.lower(), body=query)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Error executing vector search, query: %s", query)
|
logger.exception("Error executing vector search, query: %s", query)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -71,7 +71,7 @@ class TableStoreVector(BaseVector):
|
||||||
table_result = result.get_result_by_table(self._table_name)
|
table_result = result.get_result_by_table(self._table_name)
|
||||||
for item in table_result:
|
for item in table_result:
|
||||||
if item.is_ok and item.row:
|
if item.is_ok and item.row:
|
||||||
kv = {k: v for k, v, t in item.row.attribute_columns}
|
kv = {k: v for k, v, _ in item.row.attribute_columns}
|
||||||
docs.append(
|
docs.append(
|
||||||
Document(
|
Document(
|
||||||
page_content=kv[Field.CONTENT_KEY.value], metadata=json.loads(kv[Field.METADATA_KEY.value])
|
page_content=kv[Field.CONTENT_KEY.value], metadata=json.loads(kv[Field.METADATA_KEY.value])
|
||||||
|
|
|
||||||
|
|
@ -23,7 +23,7 @@ class UnstructuredWordExtractor(BaseExtractor):
|
||||||
unstructured_version = tuple(int(x) for x in __unstructured_version__.split("."))
|
unstructured_version = tuple(int(x) for x in __unstructured_version__.split("."))
|
||||||
# check the file extension
|
# check the file extension
|
||||||
try:
|
try:
|
||||||
import magic # noqa: F401
|
import magic # noqa: F401 # pyright: ignore[reportUnusedImport]
|
||||||
|
|
||||||
is_doc = detect_filetype(self._file_path) == FileType.DOC
|
is_doc = detect_filetype(self._file_path) == FileType.DOC
|
||||||
except ImportError:
|
except ImportError:
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,7 @@ class QAIndexProcessor(BaseIndexProcessor):
|
||||||
# Skip the first row
|
# Skip the first row
|
||||||
df = pd.read_csv(file)
|
df = pd.read_csv(file)
|
||||||
text_docs = []
|
text_docs = []
|
||||||
for index, row in df.iterrows():
|
for _, row in df.iterrows():
|
||||||
data = Document(page_content=row.iloc[0], metadata={"answer": row.iloc[1]})
|
data = Document(page_content=row.iloc[0], metadata={"answer": row.iloc[1]})
|
||||||
text_docs.append(data)
|
text_docs.append(data)
|
||||||
if len(text_docs) == 0:
|
if len(text_docs) == 0:
|
||||||
|
|
@ -183,7 +183,7 @@ class QAIndexProcessor(BaseIndexProcessor):
|
||||||
qa_document.metadata["doc_hash"] = hash
|
qa_document.metadata["doc_hash"] = hash
|
||||||
qa_documents.append(qa_document)
|
qa_documents.append(qa_document)
|
||||||
format_documents.extend(qa_documents)
|
format_documents.extend(qa_documents)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to format qa document")
|
logger.exception("Failed to format qa document")
|
||||||
|
|
||||||
all_qa_documents.extend(format_documents)
|
all_qa_documents.extend(format_documents)
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,6 @@ from typing import Any, Optional, Union, cast
|
||||||
from flask import Flask, current_app
|
from flask import Flask, current_app
|
||||||
from sqlalchemy import Float, and_, or_, select, text
|
from sqlalchemy import Float, and_, or_, select, text
|
||||||
from sqlalchemy import cast as sqlalchemy_cast
|
from sqlalchemy import cast as sqlalchemy_cast
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from core.app.app_config.entities import (
|
from core.app.app_config.entities import (
|
||||||
DatasetEntity,
|
DatasetEntity,
|
||||||
|
|
@ -526,7 +525,7 @@ class DatasetRetrieval:
|
||||||
)
|
)
|
||||||
child_chunk = db.session.scalar(child_chunk_stmt)
|
child_chunk = db.session.scalar(child_chunk_stmt)
|
||||||
if child_chunk:
|
if child_chunk:
|
||||||
segment = (
|
_ = (
|
||||||
db.session.query(DocumentSegment)
|
db.session.query(DocumentSegment)
|
||||||
.where(DocumentSegment.id == child_chunk.segment_id)
|
.where(DocumentSegment.id == child_chunk.segment_id)
|
||||||
.update(
|
.update(
|
||||||
|
|
@ -593,9 +592,8 @@ class DatasetRetrieval:
|
||||||
metadata_condition: Optional[MetadataCondition] = None,
|
metadata_condition: Optional[MetadataCondition] = None,
|
||||||
):
|
):
|
||||||
with flask_app.app_context():
|
with flask_app.app_context():
|
||||||
with Session(db.engine) as session:
|
dataset_stmt = select(Dataset).where(Dataset.id == dataset_id)
|
||||||
dataset_stmt = select(Dataset).where(Dataset.id == dataset_id)
|
dataset = db.session.scalar(dataset_stmt)
|
||||||
dataset = session.scalar(dataset_stmt)
|
|
||||||
|
|
||||||
if not dataset:
|
if not dataset:
|
||||||
return []
|
return []
|
||||||
|
|
@ -987,7 +985,7 @@ class DatasetRetrieval:
|
||||||
)
|
)
|
||||||
|
|
||||||
# handle invoke result
|
# handle invoke result
|
||||||
result_text, usage = self._handle_invoke_result(invoke_result=invoke_result)
|
result_text, _ = self._handle_invoke_result(invoke_result=invoke_result)
|
||||||
|
|
||||||
result_text_json = parse_and_check_json_markdown(result_text, [])
|
result_text_json = parse_and_check_json_markdown(result_text, [])
|
||||||
automatic_metadata_filters = []
|
automatic_metadata_filters = []
|
||||||
|
|
@ -1002,7 +1000,7 @@ class DatasetRetrieval:
|
||||||
"condition": item.get("comparison_operator"),
|
"condition": item.get("comparison_operator"),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
return automatic_metadata_filters
|
return automatic_metadata_filters
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -19,5 +19,5 @@ class StructuredChatOutputParser:
|
||||||
return ReactAction(response["action"], response.get("action_input", {}), text)
|
return ReactAction(response["action"], response.get("action_input", {}), text)
|
||||||
else:
|
else:
|
||||||
return ReactFinish({"output": text}, text)
|
return ReactFinish({"output": text}, text)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
raise ValueError(f"Could not parse LLM output: {text}")
|
raise ValueError(f"Could not parse LLM output: {text}")
|
||||||
|
|
|
||||||
|
|
@ -38,5 +38,5 @@ class FunctionCallMultiDatasetRouter:
|
||||||
# get retrieval model config
|
# get retrieval model config
|
||||||
return result.message.tool_calls[0].function.name
|
return result.message.tool_calls[0].function.name
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
|
||||||
|
|
@ -77,7 +77,7 @@ class ReactMultiDatasetRouter:
|
||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _react_invoke(
|
def _react_invoke(
|
||||||
|
|
@ -120,7 +120,7 @@ class ReactMultiDatasetRouter:
|
||||||
memory=None,
|
memory=None,
|
||||||
model_config=model_config,
|
model_config=model_config,
|
||||||
)
|
)
|
||||||
result_text, usage = self._invoke_llm(
|
result_text, _ = self._invoke_llm(
|
||||||
completion_param=model_config.parameters,
|
completion_param=model_config.parameters,
|
||||||
model_instance=model_instance,
|
model_instance=model_instance,
|
||||||
prompt_messages=prompt_messages,
|
prompt_messages=prompt_messages,
|
||||||
|
|
|
||||||
|
|
@ -119,7 +119,7 @@ class CeleryWorkflowExecutionRepository(WorkflowExecutionRepository):
|
||||||
|
|
||||||
logger.debug("Queued async save for workflow execution: %s", execution.id_)
|
logger.debug("Queued async save for workflow execution: %s", execution.id_)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to queue save operation for execution %s", execution.id_)
|
logger.exception("Failed to queue save operation for execution %s", execution.id_)
|
||||||
# In case of Celery failure, we could implement a fallback to synchronous save
|
# In case of Celery failure, we could implement a fallback to synchronous save
|
||||||
# For now, we'll re-raise the exception
|
# For now, we'll re-raise the exception
|
||||||
|
|
|
||||||
|
|
@ -142,7 +142,7 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||||
|
|
||||||
logger.debug("Cached and queued async save for workflow node execution: %s", execution.id)
|
logger.debug("Cached and queued async save for workflow node execution: %s", execution.id)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to cache or queue save operation for node execution %s", execution.id)
|
logger.exception("Failed to cache or queue save operation for node execution %s", execution.id)
|
||||||
# In case of Celery failure, we could implement a fallback to synchronous save
|
# In case of Celery failure, we could implement a fallback to synchronous save
|
||||||
# For now, we'll re-raise the exception
|
# For now, we'll re-raise the exception
|
||||||
|
|
@ -185,6 +185,6 @@ class CeleryWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository):
|
||||||
logger.debug("Retrieved %d workflow node executions for run %s from cache", len(result), workflow_run_id)
|
logger.debug("Retrieved %d workflow node executions for run %s from cache", len(result), workflow_run_id)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to get workflow node executions for run %s from cache", workflow_run_id)
|
logger.exception("Failed to get workflow node executions for run %s from cache", workflow_run_id)
|
||||||
return []
|
return []
|
||||||
|
|
|
||||||
|
|
@ -250,7 +250,7 @@ class SQLAlchemyWorkflowNodeExecutionRepository(WorkflowNodeExecutionRepository)
|
||||||
logger.debug("Updating cache for node_execution_id: %s", db_model.node_execution_id)
|
logger.debug("Updating cache for node_execution_id: %s", db_model.node_execution_id)
|
||||||
self._node_execution_cache[db_model.node_execution_id] = db_model
|
self._node_execution_cache[db_model.node_execution_id] = db_model
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to save workflow node execution after all retries")
|
logger.exception("Failed to save workflow node execution after all retries")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -191,7 +191,7 @@ class ApiToolProviderController(ToolProviderController):
|
||||||
self.tools = tools
|
self.tools = tools
|
||||||
return tools
|
return tools
|
||||||
|
|
||||||
def get_tool(self, tool_name: str):
|
def get_tool(self, tool_name: str) -> ApiTool:
|
||||||
"""
|
"""
|
||||||
get tool by name
|
get tool by name
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -107,5 +107,5 @@ default_tool_label_dict = {
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
default_tool_labels = [v for k, v in default_tool_label_dict.items()]
|
default_tool_labels = list(default_tool_label_dict.values())
|
||||||
default_tool_label_name_list = [label.name for label in default_tool_labels]
|
default_tool_label_name_list = [label.name for label in default_tool_labels]
|
||||||
|
|
|
||||||
|
|
@ -303,16 +303,13 @@ class ToolManager:
|
||||||
tenant_id=tenant_id,
|
tenant_id=tenant_id,
|
||||||
controller=api_provider,
|
controller=api_provider,
|
||||||
)
|
)
|
||||||
return cast(
|
return api_provider.get_tool(tool_name).fork_tool_runtime(
|
||||||
ApiTool,
|
runtime=ToolRuntime(
|
||||||
api_provider.get_tool(tool_name).fork_tool_runtime(
|
tenant_id=tenant_id,
|
||||||
runtime=ToolRuntime(
|
credentials=encrypter.decrypt(credentials),
|
||||||
tenant_id=tenant_id,
|
invoke_from=invoke_from,
|
||||||
credentials=encrypter.decrypt(credentials),
|
tool_invoke_from=tool_invoke_from,
|
||||||
invoke_from=invoke_from,
|
)
|
||||||
tool_invoke_from=tool_invoke_from,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
elif provider_type == ToolProviderType.WORKFLOW:
|
elif provider_type == ToolProviderType.WORKFLOW:
|
||||||
workflow_provider_stmt = select(WorkflowToolProvider).where(
|
workflow_provider_stmt = select(WorkflowToolProvider).where(
|
||||||
|
|
|
||||||
|
|
@ -68,7 +68,7 @@ class AnswerStreamProcessor(StreamProcessor):
|
||||||
|
|
||||||
def reset(self) -> None:
|
def reset(self) -> None:
|
||||||
self.route_position = {}
|
self.route_position = {}
|
||||||
for answer_node_id, route_chunks in self.generate_routes.answer_generate_route.items():
|
for answer_node_id, _ in self.generate_routes.answer_generate_route.items():
|
||||||
self.route_position[answer_node_id] = 0
|
self.route_position[answer_node_id] = 0
|
||||||
self.rest_node_ids = self.graph.node_ids.copy()
|
self.rest_node_ids = self.graph.node_ids.copy()
|
||||||
self.current_stream_chunk_generating_node_ids = {}
|
self.current_stream_chunk_generating_node_ids = {}
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ import logging
|
||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
from collections.abc import Mapping, Sequence
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import Any, Optional, cast
|
from typing import Any, Optional
|
||||||
|
|
||||||
import chardet
|
import chardet
|
||||||
import docx
|
import docx
|
||||||
|
|
@ -428,9 +428,9 @@ def _download_file_content(file: File) -> bytes:
|
||||||
raise FileDownloadError("Missing URL for remote file")
|
raise FileDownloadError("Missing URL for remote file")
|
||||||
response = ssrf_proxy.get(file.remote_url)
|
response = ssrf_proxy.get(file.remote_url)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return cast(bytes, response.content)
|
return response.content
|
||||||
else:
|
else:
|
||||||
return cast(bytes, file_manager.download(file))
|
return file_manager.download(file)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise FileDownloadError(f"Error downloading file: {str(e)}") from e
|
raise FileDownloadError(f"Error downloading file: {str(e)}") from e
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -571,7 +571,7 @@ class KnowledgeRetrievalNode(BaseNode):
|
||||||
"condition": item.get("comparison_operator"),
|
"condition": item.get("comparison_operator"),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
return automatic_metadata_filters
|
return automatic_metadata_filters
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -324,7 +324,7 @@ class LoopNode(BaseNode):
|
||||||
|
|
||||||
# Process conditions if at least one variable is available
|
# Process conditions if at least one variable is available
|
||||||
if available_conditions:
|
if available_conditions:
|
||||||
input_conditions, group_result, check_break_result = condition_processor.process_conditions(
|
_, _, check_break_result = condition_processor.process_conditions(
|
||||||
variable_pool=self.graph_runtime_state.variable_pool,
|
variable_pool=self.graph_runtime_state.variable_pool,
|
||||||
conditions=available_conditions,
|
conditions=available_conditions,
|
||||||
operator=logical_operator,
|
operator=logical_operator,
|
||||||
|
|
|
||||||
|
|
@ -43,10 +43,6 @@ def _validate_type(parameter_type: str) -> SegmentType:
|
||||||
return SegmentType(parameter_type)
|
return SegmentType(parameter_type)
|
||||||
|
|
||||||
|
|
||||||
class _ParameterConfigError(Exception):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class ParameterConfig(BaseModel):
|
class ParameterConfig(BaseModel):
|
||||||
"""
|
"""
|
||||||
Parameter Config.
|
Parameter Config.
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ _T = TypeVar("_T", bound=MutableMapping[str, Any])
|
||||||
def variable_to_processed_data(selector: Sequence[str], seg: Segment) -> UpdatedVariable:
|
def variable_to_processed_data(selector: Sequence[str], seg: Segment) -> UpdatedVariable:
|
||||||
if len(selector) < SELECTORS_LENGTH:
|
if len(selector) < SELECTORS_LENGTH:
|
||||||
raise Exception("selector too short")
|
raise Exception("selector too short")
|
||||||
node_id, var_name = selector[:2]
|
_, var_name = selector[:2]
|
||||||
return UpdatedVariable(
|
return UpdatedVariable(
|
||||||
name=var_name,
|
name=var_name,
|
||||||
selector=list(selector[:2]),
|
selector=list(selector[:2]),
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,30 @@
|
||||||
from .clean_when_dataset_deleted import handle
|
from .clean_when_dataset_deleted import handle as handle_clean_when_dataset_deleted
|
||||||
from .clean_when_document_deleted import handle
|
from .clean_when_document_deleted import handle as handle_clean_when_document_deleted
|
||||||
from .create_document_index import handle
|
from .create_document_index import handle as handle_create_document_index
|
||||||
from .create_installed_app_when_app_created import handle
|
from .create_installed_app_when_app_created import handle as handle_create_installed_app_when_app_created
|
||||||
from .create_site_record_when_app_created import handle
|
from .create_site_record_when_app_created import handle as handle_create_site_record_when_app_created
|
||||||
from .delete_tool_parameters_cache_when_sync_draft_workflow import handle
|
from .delete_tool_parameters_cache_when_sync_draft_workflow import (
|
||||||
from .update_app_dataset_join_when_app_model_config_updated import handle
|
handle as handle_delete_tool_parameters_cache_when_sync_draft_workflow,
|
||||||
from .update_app_dataset_join_when_app_published_workflow_updated import handle
|
)
|
||||||
|
from .update_app_dataset_join_when_app_model_config_updated import (
|
||||||
|
handle as handle_update_app_dataset_join_when_app_model_config_updated,
|
||||||
|
)
|
||||||
|
from .update_app_dataset_join_when_app_published_workflow_updated import (
|
||||||
|
handle as handle_update_app_dataset_join_when_app_published_workflow_updated,
|
||||||
|
)
|
||||||
|
|
||||||
# Consolidated handler replaces both deduct_quota_when_message_created and
|
# Consolidated handler replaces both deduct_quota_when_message_created and
|
||||||
# update_provider_last_used_at_when_message_created
|
# update_provider_last_used_at_when_message_created
|
||||||
from .update_provider_when_message_created import handle
|
from .update_provider_when_message_created import handle as handle_update_provider_when_message_created
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"handle_clean_when_dataset_deleted",
|
||||||
|
"handle_clean_when_document_deleted",
|
||||||
|
"handle_create_document_index",
|
||||||
|
"handle_create_installed_app_when_app_created",
|
||||||
|
"handle_create_site_record_when_app_created",
|
||||||
|
"handle_delete_tool_parameters_cache_when_sync_draft_workflow",
|
||||||
|
"handle_update_app_dataset_join_when_app_model_config_updated",
|
||||||
|
"handle_update_app_dataset_join_when_app_published_workflow_updated",
|
||||||
|
"handle_update_provider_when_message_created",
|
||||||
|
]
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ def get_dataset_ids_from_workflow(published_workflow: Workflow) -> set[str]:
|
||||||
try:
|
try:
|
||||||
node_data = KnowledgeRetrievalNodeData(**node.get("data", {}))
|
node_data = KnowledgeRetrievalNodeData(**node.get("data", {}))
|
||||||
dataset_ids.update(dataset_id for dataset_id in node_data.dataset_ids)
|
dataset_ids.update(dataset_id for dataset_id in node_data.dataset_ids)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
return dataset_ids
|
return dataset_ids
|
||||||
|
|
|
||||||
|
|
@ -204,7 +204,7 @@ def _calculate_quota_usage(
|
||||||
elif quota_unit == QuotaUnit.TIMES:
|
elif quota_unit == QuotaUnit.TIMES:
|
||||||
return 1
|
return 1
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to calculate quota usage")
|
logger.exception("Failed to calculate quota usage")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ def init_app(app: DifyApp):
|
||||||
|
|
||||||
def before_send(event, hint):
|
def before_send(event, hint):
|
||||||
if "exc_info" in hint:
|
if "exc_info" in hint:
|
||||||
exc_type, exc_value, tb = hint["exc_info"]
|
_, exc_value, _ = hint["exc_info"]
|
||||||
if parse_error.defaultErrorResponse in str(exc_value):
|
if parse_error.defaultErrorResponse in str(exc_value):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -139,7 +139,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||||
schema=self._config.schema_name,
|
schema=self._config.schema_name,
|
||||||
)
|
)
|
||||||
logger.debug("ClickZetta connection established")
|
logger.debug("ClickZetta connection established")
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to connect to ClickZetta")
|
logger.exception("Failed to connect to ClickZetta")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
@ -150,7 +150,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||||
self._connection, self._config.volume_type, self._config.volume_name
|
self._connection, self._config.volume_type, self._config.volume_name
|
||||||
)
|
)
|
||||||
logger.debug("Permission manager initialized")
|
logger.debug("Permission manager initialized")
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to initialize permission manager")
|
logger.exception("Failed to initialize permission manager")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
@ -213,7 +213,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||||
if fetch:
|
if fetch:
|
||||||
return cursor.fetchall()
|
return cursor.fetchall()
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("SQL execution failed: %s", sql)
|
logger.exception("SQL execution failed: %s", sql)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
@ -349,7 +349,7 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||||
|
|
||||||
# Find the downloaded file (may be in subdirectories)
|
# Find the downloaded file (may be in subdirectories)
|
||||||
downloaded_file = None
|
downloaded_file = None
|
||||||
for root, dirs, files in os.walk(temp_dir):
|
for root, _, files in os.walk(temp_dir):
|
||||||
for file in files:
|
for file in files:
|
||||||
if file == filename or file == os.path.basename(filename):
|
if file == filename or file == os.path.basename(filename):
|
||||||
downloaded_file = Path(root) / file
|
downloaded_file = Path(root) / file
|
||||||
|
|
@ -524,6 +524,6 @@ class ClickZettaVolumeStorage(BaseStorage):
|
||||||
logger.debug("Scanned %d items in path %s", len(result), path)
|
logger.debug("Scanned %d items in path %s", len(result), path)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Error scanning path %s", path)
|
logger.exception("Error scanning path %s", path)
|
||||||
return []
|
return []
|
||||||
|
|
|
||||||
|
|
@ -145,7 +145,7 @@ class FileLifecycleManager:
|
||||||
logger.info("File %s saved with lifecycle management, version %s", filename, new_version)
|
logger.info("File %s saved with lifecycle management, version %s", filename, new_version)
|
||||||
return file_metadata
|
return file_metadata
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to save file with lifecycle")
|
logger.exception("Failed to save file with lifecycle")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
@ -163,7 +163,7 @@ class FileLifecycleManager:
|
||||||
if filename in metadata_dict:
|
if filename in metadata_dict:
|
||||||
return FileMetadata.from_dict(metadata_dict[filename])
|
return FileMetadata.from_dict(metadata_dict[filename])
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to get file metadata for %s", filename)
|
logger.exception("Failed to get file metadata for %s", filename)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
@ -192,7 +192,7 @@ class FileLifecycleManager:
|
||||||
# Parse version number
|
# Parse version number
|
||||||
version_str = file_path.split(".v")[-1].split(".")[0]
|
version_str = file_path.split(".v")[-1].split(".")[0]
|
||||||
try:
|
try:
|
||||||
version_num = int(version_str)
|
_ = int(version_str)
|
||||||
# Simplified processing here, should actually read metadata from version file
|
# Simplified processing here, should actually read metadata from version file
|
||||||
# Temporarily create basic metadata information
|
# Temporarily create basic metadata information
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
|
@ -203,7 +203,7 @@ class FileLifecycleManager:
|
||||||
|
|
||||||
return sorted(versions, key=lambda x: x.version or 0, reverse=True)
|
return sorted(versions, key=lambda x: x.version or 0, reverse=True)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to list file versions for %s", filename)
|
logger.exception("Failed to list file versions for %s", filename)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
@ -237,7 +237,7 @@ class FileLifecycleManager:
|
||||||
self.save_with_lifecycle(filename, version_data, {"restored_from": str(version)})
|
self.save_with_lifecycle(filename, version_data, {"restored_from": str(version)})
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to restore %s to version %s", filename, version)
|
logger.exception("Failed to restore %s to version %s", filename, version)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -270,7 +270,7 @@ class FileLifecycleManager:
|
||||||
logger.info("File %s archived successfully", filename)
|
logger.info("File %s archived successfully", filename)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to archive file %s", filename)
|
logger.exception("Failed to archive file %s", filename)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -314,7 +314,7 @@ class FileLifecycleManager:
|
||||||
logger.info("File %s soft deleted successfully", filename)
|
logger.info("File %s soft deleted successfully", filename)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to soft delete file %s", filename)
|
logger.exception("Failed to soft delete file %s", filename)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -372,7 +372,7 @@ class FileLifecycleManager:
|
||||||
|
|
||||||
return cleaned_count
|
return cleaned_count
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to cleanup old versions")
|
logger.exception("Failed to cleanup old versions")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
@ -427,7 +427,7 @@ class FileLifecycleManager:
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to get storage statistics")
|
logger.exception("Failed to get storage statistics")
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
|
|
@ -465,7 +465,7 @@ class FileLifecycleManager:
|
||||||
metadata_content = json.dumps(metadata_dict, indent=2, ensure_ascii=False)
|
metadata_content = json.dumps(metadata_dict, indent=2, ensure_ascii=False)
|
||||||
self._storage.save(self._metadata_file, metadata_content.encode("utf-8"))
|
self._storage.save(self._metadata_file, metadata_content.encode("utf-8"))
|
||||||
logger.debug("Metadata saved successfully")
|
logger.debug("Metadata saved successfully")
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to save metadata")
|
logger.exception("Failed to save metadata")
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
@ -508,7 +508,7 @@ class FileLifecycleManager:
|
||||||
result = self._permission_manager.validate_operation(mapped_operation, self._dataset_id)
|
result = self._permission_manager.validate_operation(mapped_operation, self._dataset_id)
|
||||||
return bool(result)
|
return bool(result)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Permission check failed for %s operation %s", filename, operation)
|
logger.exception("Permission check failed for %s operation %s", filename, operation)
|
||||||
# Safe default: deny access when permission check fails
|
# Safe default: deny access when permission check fails
|
||||||
return False
|
return False
|
||||||
|
|
|
||||||
|
|
@ -84,7 +84,7 @@ class VolumePermissionManager:
|
||||||
logger.warning("Unknown volume type: %s", self._volume_type)
|
logger.warning("Unknown volume type: %s", self._volume_type)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Permission check failed")
|
logger.exception("Permission check failed")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -119,7 +119,7 @@ class VolumePermissionManager:
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("User Volume permission check failed")
|
logger.exception("User Volume permission check failed")
|
||||||
# For User Volume, if permission check fails, it might be a configuration issue, provide friendlier error message
|
# For User Volume, if permission check fails, it might be a configuration issue, provide friendlier error message
|
||||||
logger.info("User Volume permission check failed, but permission checking is disabled in this version")
|
logger.info("User Volume permission check failed, but permission checking is disabled in this version")
|
||||||
|
|
@ -158,7 +158,7 @@ class VolumePermissionManager:
|
||||||
|
|
||||||
return has_permission
|
return has_permission
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Table volume permission check failed for %s", table_name)
|
logger.exception("Table volume permission check failed for %s", table_name)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
@ -216,7 +216,7 @@ class VolumePermissionManager:
|
||||||
|
|
||||||
return has_permission
|
return has_permission
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("External volume permission check failed for %s", self._volume_name)
|
logger.exception("External volume permission check failed for %s", self._volume_name)
|
||||||
logger.info("External Volume permission check failed, but permission checking is disabled in this version")
|
logger.info("External Volume permission check failed, but permission checking is disabled in this version")
|
||||||
return False
|
return False
|
||||||
|
|
@ -292,7 +292,7 @@ class VolumePermissionManager:
|
||||||
if result:
|
if result:
|
||||||
self._current_username = result[0]
|
self._current_username = result[0]
|
||||||
return str(self._current_username)
|
return str(self._current_username)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to get current username")
|
logger.exception("Failed to get current username")
|
||||||
|
|
||||||
return "unknown"
|
return "unknown"
|
||||||
|
|
@ -316,7 +316,7 @@ class VolumePermissionManager:
|
||||||
for grant in grants:
|
for grant in grants:
|
||||||
if len(grant) >= 3: # Typical format: (privilege, object_type, object_name, ...)
|
if len(grant) >= 3: # Typical format: (privilege, object_type, object_name, ...)
|
||||||
privilege = grant[0].upper()
|
privilege = grant[0].upper()
|
||||||
object_type = grant[1].upper() if len(grant) > 1 else ""
|
_ = grant[1].upper() if len(grant) > 1 else ""
|
||||||
|
|
||||||
# Collect all relevant permissions
|
# Collect all relevant permissions
|
||||||
if privilege in ["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]:
|
if privilege in ["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]:
|
||||||
|
|
@ -521,7 +521,7 @@ class VolumePermissionManager:
|
||||||
logger.warning("Unknown volume type for permission inheritance: %s", self._volume_type)
|
logger.warning("Unknown volume type for permission inheritance: %s", self._volume_type)
|
||||||
return False
|
return False
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Permission inheritance check failed")
|
logger.exception("Permission inheritance check failed")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -185,7 +185,7 @@ def timezone(timezone_string):
|
||||||
def generate_string(n):
|
def generate_string(n):
|
||||||
letters_digits = string.ascii_letters + string.digits
|
letters_digits = string.ascii_letters + string.digits
|
||||||
result = ""
|
result = ""
|
||||||
for i in range(n):
|
for _ in range(n):
|
||||||
result += secrets.choice(letters_digits)
|
result += secrets.choice(letters_digits)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
|
||||||
|
|
@ -33,15 +33,15 @@ class SendGridClient:
|
||||||
logger.debug(response.body)
|
logger.debug(response.body)
|
||||||
logger.debug(response.headers)
|
logger.debug(response.headers)
|
||||||
|
|
||||||
except TimeoutError as e:
|
except TimeoutError:
|
||||||
logger.exception("SendGridClient Timeout occurred while sending email")
|
logger.exception("SendGridClient Timeout occurred while sending email")
|
||||||
raise
|
raise
|
||||||
except (UnauthorizedError, ForbiddenError) as e:
|
except (UnauthorizedError, ForbiddenError):
|
||||||
logger.exception(
|
logger.exception(
|
||||||
"SendGridClient Authentication failed. "
|
"SendGridClient Authentication failed. "
|
||||||
"Verify that your credentials and the 'from' email address are correct"
|
"Verify that your credentials and the 'from' email address are correct"
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("SendGridClient Unexpected error occurred while sending email to %s", _to)
|
logger.exception("SendGridClient Unexpected error occurred while sending email to %s", _to)
|
||||||
raise
|
raise
|
||||||
|
|
|
||||||
|
|
@ -45,13 +45,13 @@ class SMTPClient:
|
||||||
msg.attach(MIMEText(mail["html"], "html"))
|
msg.attach(MIMEText(mail["html"], "html"))
|
||||||
|
|
||||||
smtp.sendmail(self._from, mail["to"], msg.as_string())
|
smtp.sendmail(self._from, mail["to"], msg.as_string())
|
||||||
except smtplib.SMTPException as e:
|
except smtplib.SMTPException:
|
||||||
logger.exception("SMTP error occurred")
|
logger.exception("SMTP error occurred")
|
||||||
raise
|
raise
|
||||||
except TimeoutError as e:
|
except TimeoutError:
|
||||||
logger.exception("Timeout occurred while sending email")
|
logger.exception("Timeout occurred while sending email")
|
||||||
raise
|
raise
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Unexpected error occurred while sending email to %s", mail["to"])
|
logger.exception("Unexpected error occurred while sending email to %s", mail["to"])
|
||||||
raise
|
raise
|
||||||
finally:
|
finally:
|
||||||
|
|
|
||||||
|
|
@ -915,7 +915,7 @@ class DatasetKeywordTable(Base):
|
||||||
if keyword_table_text:
|
if keyword_table_text:
|
||||||
return json.loads(keyword_table_text.decode("utf-8"), cls=SetDecoder)
|
return json.loads(keyword_table_text.decode("utf-8"), cls=SetDecoder)
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to load keyword table from file: %s", file_key)
|
logger.exception("Failed to load keyword table from file: %s", file_key)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -111,7 +111,7 @@ dev = [
|
||||||
"faker~=32.1.0",
|
"faker~=32.1.0",
|
||||||
"lxml-stubs~=0.5.1",
|
"lxml-stubs~=0.5.1",
|
||||||
"ty~=0.0.1a19",
|
"ty~=0.0.1a19",
|
||||||
"mypy~=1.17.1",
|
"basedpyright~=1.31.0",
|
||||||
"ruff~=0.12.3",
|
"ruff~=0.12.3",
|
||||||
"pytest~=8.3.2",
|
"pytest~=8.3.2",
|
||||||
"pytest-benchmark~=4.0.0",
|
"pytest-benchmark~=4.0.0",
|
||||||
|
|
@ -218,4 +218,3 @@ vdb = [
|
||||||
"xinference-client~=1.2.2",
|
"xinference-client~=1.2.2",
|
||||||
"mo-vector~=0.1.13",
|
"mo-vector~=0.1.13",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
|
||||||
48
api/pyrightconfig.json
Normal file
48
api/pyrightconfig.json
Normal file
|
|
@ -0,0 +1,48 @@
|
||||||
|
{
|
||||||
|
"include": ["."],
|
||||||
|
"exclude": ["tests/", "migrations/", ".venv/"],
|
||||||
|
"typeCheckingMode": "strict",
|
||||||
|
"pythonVersion": "3.11",
|
||||||
|
"pythonPlatform": "All",
|
||||||
|
"reportMissingTypeStubs": false,
|
||||||
|
"reportGeneralTypeIssues": "none",
|
||||||
|
"reportOptionalMemberAccess": "none",
|
||||||
|
"reportOptionalIterable": "none",
|
||||||
|
"reportOptionalOperand": "none",
|
||||||
|
"reportOptionalSubscript": "none",
|
||||||
|
"reportTypedDictNotRequiredAccess": "none",
|
||||||
|
"reportPrivateImportUsage": "none",
|
||||||
|
"reportUnsupportedDunderAll": "none",
|
||||||
|
"reportUnnecessaryTypeIgnoreComment": "none",
|
||||||
|
"reportMatchNotExhaustive": "none",
|
||||||
|
"reportImplicitOverride": "none",
|
||||||
|
"reportCallInDefaultInitializer": "none",
|
||||||
|
"reportUnnecessaryIsInstance": "none",
|
||||||
|
"reportUnnecessaryComparison": "none",
|
||||||
|
"reportUnknownParameterType": "none",
|
||||||
|
"reportMissingParameterType": "none",
|
||||||
|
"reportUnknownArgumentType": "none",
|
||||||
|
"reportUnknownVariableType": "none",
|
||||||
|
"reportUnknownMemberType": "none",
|
||||||
|
"reportMissingTypeArgument": "none",
|
||||||
|
"reportUntypedFunctionDecorator": "none",
|
||||||
|
"reportUnknownLambdaType": "none",
|
||||||
|
"reportPrivateUsage": "none",
|
||||||
|
"reportConstantRedefinition": "none",
|
||||||
|
"reportIncompatibleMethodOverride": "none",
|
||||||
|
"reportIncompatibleVariableOverride": "none",
|
||||||
|
"reportOverlappingOverload": "none",
|
||||||
|
"reportPossiblyUnboundVariable": "none",
|
||||||
|
"reportUnusedImport": "none",
|
||||||
|
"reportUnusedFunction": "none",
|
||||||
|
"reportArgumentType": "none",
|
||||||
|
"reportAssignmentType": "none",
|
||||||
|
"reportAttributeAccessIssue": "none",
|
||||||
|
"reportCallIssue": "none",
|
||||||
|
"reportIndexIssue": "none",
|
||||||
|
"reportRedeclaration": "none",
|
||||||
|
"reportReturnType": "none",
|
||||||
|
"reportOperatorIssue": "none",
|
||||||
|
"reportTypeCommentUsage": "none",
|
||||||
|
"reportDeprecated": "none"
|
||||||
|
}
|
||||||
|
|
@ -22,7 +22,7 @@ Implementation Notes:
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional, cast
|
from typing import Optional
|
||||||
|
|
||||||
from sqlalchemy import delete, select
|
from sqlalchemy import delete, select
|
||||||
from sqlalchemy.orm import Session, sessionmaker
|
from sqlalchemy.orm import Session, sessionmaker
|
||||||
|
|
@ -117,7 +117,7 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
|
||||||
WorkflowRun.app_id == app_id,
|
WorkflowRun.app_id == app_id,
|
||||||
WorkflowRun.id == run_id,
|
WorkflowRun.id == run_id,
|
||||||
)
|
)
|
||||||
return cast(Optional[WorkflowRun], session.scalar(stmt))
|
return session.scalar(stmt)
|
||||||
|
|
||||||
def get_expired_runs_batch(
|
def get_expired_runs_batch(
|
||||||
self,
|
self,
|
||||||
|
|
@ -137,7 +137,7 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
|
||||||
)
|
)
|
||||||
.limit(batch_size)
|
.limit(batch_size)
|
||||||
)
|
)
|
||||||
return cast(Sequence[WorkflowRun], session.scalars(stmt).all())
|
return session.scalars(stmt).all()
|
||||||
|
|
||||||
def delete_runs_by_ids(
|
def delete_runs_by_ids(
|
||||||
self,
|
self,
|
||||||
|
|
@ -154,7 +154,7 @@ class DifyAPISQLAlchemyWorkflowRunRepository(APIWorkflowRunRepository):
|
||||||
result = session.execute(stmt)
|
result = session.execute(stmt)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
deleted_count = cast(int, result.rowcount)
|
deleted_count = result.rowcount
|
||||||
logger.info("Deleted %s workflow runs by IDs", deleted_count)
|
logger.info("Deleted %s workflow runs by IDs", deleted_count)
|
||||||
return deleted_count
|
return deleted_count
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -77,7 +77,7 @@ def clean_workflow_runlogs_precise():
|
||||||
|
|
||||||
logger.info("Cleanup completed: %s expired workflow run logs deleted", total_deleted)
|
logger.info("Cleanup completed: %s expired workflow run logs deleted", total_deleted)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
db.session.rollback()
|
db.session.rollback()
|
||||||
logger.exception("Unexpected error in workflow log cleanup")
|
logger.exception("Unexpected error in workflow log cleanup")
|
||||||
raise
|
raise
|
||||||
|
|
@ -149,7 +149,7 @@ def _delete_batch_with_retry(workflow_run_ids: list[str], attempt_count: int) ->
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
db.session.rollback()
|
db.session.rollback()
|
||||||
logger.exception("Batch deletion failed (attempt %s)", attempt_count + 1)
|
logger.exception("Batch deletion failed (attempt %s)", attempt_count + 1)
|
||||||
return False
|
return False
|
||||||
|
|
|
||||||
|
|
@ -63,10 +63,10 @@ def queue_monitor_task():
|
||||||
"alert_time": current_time,
|
"alert_time": current_time,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(click.style("Exception occurred during sending email", fg="red"))
|
logger.exception(click.style("Exception occurred during sending email", fg="red"))
|
||||||
|
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception(click.style("Exception occurred during queue monitoring", fg="red"))
|
logger.exception(click.style("Exception occurred during queue monitoring", fg="red"))
|
||||||
finally:
|
finally:
|
||||||
if db.session.is_active:
|
if db.session.is_active:
|
||||||
|
|
|
||||||
|
|
@ -330,7 +330,7 @@ class AppAnnotationService:
|
||||||
# Skip the first row
|
# Skip the first row
|
||||||
df = pd.read_csv(file, dtype=str)
|
df = pd.read_csv(file, dtype=str)
|
||||||
result = []
|
result = []
|
||||||
for index, row in df.iterrows():
|
for _, row in df.iterrows():
|
||||||
content = {"question": row.iloc[0], "answer": row.iloc[1]}
|
content = {"question": row.iloc[0], "answer": row.iloc[1]}
|
||||||
result.append(content)
|
result.append(content)
|
||||||
if len(result) == 0:
|
if len(result) == 0:
|
||||||
|
|
|
||||||
|
|
@ -227,7 +227,7 @@ class AppGenerateService:
|
||||||
# If workflow_id is specified, get the specific workflow version
|
# If workflow_id is specified, get the specific workflow version
|
||||||
if workflow_id:
|
if workflow_id:
|
||||||
try:
|
try:
|
||||||
workflow_uuid = uuid.UUID(workflow_id)
|
_ = uuid.UUID(workflow_id)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
raise WorkflowIdFormatError(f"Invalid workflow_id format: '{workflow_id}'. ")
|
raise WorkflowIdFormatError(f"Invalid workflow_id format: '{workflow_id}'. ")
|
||||||
workflow = workflow_service.get_published_workflow_by_id(app_model=app_model, workflow_id=workflow_id)
|
workflow = workflow_service.get_published_workflow_by_id(app_model=app_model, workflow_id=workflow_id)
|
||||||
|
|
|
||||||
|
|
@ -96,7 +96,7 @@ class AppService:
|
||||||
)
|
)
|
||||||
except (ProviderTokenNotInitError, LLMBadRequestError):
|
except (ProviderTokenNotInitError, LLMBadRequestError):
|
||||||
model_instance = None
|
model_instance = None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Get default model instance failed, tenant_id: %s", tenant_id)
|
logger.exception("Get default model instance failed, tenant_id: %s", tenant_id)
|
||||||
model_instance = None
|
model_instance = None
|
||||||
|
|
||||||
|
|
@ -201,7 +201,7 @@ class AppService:
|
||||||
|
|
||||||
# override tool parameters
|
# override tool parameters
|
||||||
tool["tool_parameters"] = masked_parameter
|
tool["tool_parameters"] = masked_parameter
|
||||||
except Exception as e:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# override agent mode
|
# override agent mode
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,7 @@ class ExternalDatasetService:
|
||||||
raise ValueError(f"invalid endpoint: {endpoint}")
|
raise ValueError(f"invalid endpoint: {endpoint}")
|
||||||
try:
|
try:
|
||||||
response = httpx.post(endpoint, headers={"Authorization": f"Bearer {api_key}"})
|
response = httpx.post(endpoint, headers={"Authorization": f"Bearer {api_key}"})
|
||||||
except Exception as e:
|
except Exception:
|
||||||
raise ValueError(f"failed to connect to the endpoint: {endpoint}")
|
raise ValueError(f"failed to connect to the endpoint: {endpoint}")
|
||||||
if response.status_code == 502:
|
if response.status_code == 502:
|
||||||
raise ValueError(f"Bad Gateway: failed to connect to the endpoint: {endpoint}")
|
raise ValueError(f"Bad Gateway: failed to connect to the endpoint: {endpoint}")
|
||||||
|
|
|
||||||
|
|
@ -175,7 +175,7 @@ limit 1000"""
|
||||||
# update jina to langgenius/jina_tool/jina etc.
|
# update jina to langgenius/jina_tool/jina etc.
|
||||||
updated_value = provider_cls(provider_name).to_string()
|
updated_value = provider_cls(provider_name).to_string()
|
||||||
batch_updates.append((updated_value, record_id))
|
batch_updates.append((updated_value, record_id))
|
||||||
except Exception as e:
|
except Exception:
|
||||||
failed_ids.append(record_id)
|
failed_ids.append(record_id)
|
||||||
click.echo(
|
click.echo(
|
||||||
click.style(
|
click.style(
|
||||||
|
|
|
||||||
|
|
@ -128,7 +128,7 @@ class ToolTransformService:
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, value in schema.items():
|
for name in schema:
|
||||||
if result.masked_credentials:
|
if result.masked_credentials:
|
||||||
result.masked_credentials[name] = ""
|
result.masked_credentials[name] = ""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ def delete_annotation_index_task(annotation_id: str, app_id: str, tenant_id: str
|
||||||
logger.exception("Delete annotation index failed when annotation deleted.")
|
logger.exception("Delete annotation index failed when annotation deleted.")
|
||||||
end_at = time.perf_counter()
|
end_at = time.perf_counter()
|
||||||
logger.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
|
logger.info(click.style(f"App annotations index deleted : {app_id} latency: {end_at - start_at}", fg="green"))
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Annotation deleted index failed")
|
logger.exception("Annotation deleted index failed")
|
||||||
finally:
|
finally:
|
||||||
db.session.close()
|
db.session.close()
|
||||||
|
|
|
||||||
|
|
@ -79,7 +79,7 @@ def batch_create_segment_to_index_task(
|
||||||
# Skip the first row
|
# Skip the first row
|
||||||
df = pd.read_csv(file_path)
|
df = pd.read_csv(file_path)
|
||||||
content = []
|
content = []
|
||||||
for index, row in df.iterrows():
|
for _, row in df.iterrows():
|
||||||
if dataset_document.doc_form == "qa_model":
|
if dataset_document.doc_form == "qa_model":
|
||||||
data = {"content": row.iloc[0], "answer": row.iloc[1]}
|
data = {"content": row.iloc[0], "answer": row.iloc[1]}
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ def clean_dataset_task(
|
||||||
index_processor = IndexProcessorFactory(doc_form).init_index_processor()
|
index_processor = IndexProcessorFactory(doc_form).init_index_processor()
|
||||||
index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True)
|
index_processor.clean(dataset, None, with_keywords=True, delete_child_chunks=True)
|
||||||
logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
|
logger.info(click.style(f"Successfully cleaned vector database for dataset: {dataset_id}", fg="green"))
|
||||||
except Exception as index_cleanup_error:
|
except Exception:
|
||||||
logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
|
logger.exception(click.style(f"Failed to clean vector database for dataset {dataset_id}", fg="red"))
|
||||||
# Continue with document and segment deletion even if vector cleanup fails
|
# Continue with document and segment deletion even if vector cleanup fails
|
||||||
logger.info(
|
logger.info(
|
||||||
|
|
@ -145,7 +145,7 @@ def clean_dataset_task(
|
||||||
try:
|
try:
|
||||||
db.session.rollback()
|
db.session.rollback()
|
||||||
logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
|
logger.info(click.style(f"Rolled back database session for dataset: {dataset_id}", fg="yellow"))
|
||||||
except Exception as rollback_error:
|
except Exception:
|
||||||
logger.exception("Failed to rollback database session")
|
logger.exception("Failed to rollback database session")
|
||||||
|
|
||||||
logger.exception("Cleaned dataset when dataset deleted failed")
|
logger.exception("Cleaned dataset when dataset deleted failed")
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ def delete_account_task(account_id):
|
||||||
account = db.session.query(Account).where(Account.id == account_id).first()
|
account = db.session.query(Account).where(Account.id == account_id).first()
|
||||||
try:
|
try:
|
||||||
BillingService.delete_account(account_id)
|
BillingService.delete_account(account_id)
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.exception("Failed to delete account %s from billing service.", account_id)
|
logger.exception("Failed to delete account %s from billing service.", account_id)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -146,7 +146,7 @@ def process_tenant_plugin_autoupgrade_check_task(
|
||||||
fg="green",
|
fg="green",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
task_start_resp = manager.upgrade_plugin(
|
_ = manager.upgrade_plugin(
|
||||||
tenant_id,
|
tenant_id,
|
||||||
original_unique_identifier,
|
original_unique_identifier,
|
||||||
new_unique_identifier,
|
new_unique_identifier,
|
||||||
|
|
|
||||||
4278
api/uv.lock
generated
4278
api/uv.lock
generated
File diff suppressed because it is too large
Load diff
9
dev/basedpyright-check
Executable file
9
dev/basedpyright-check
Executable file
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -x
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(dirname "$(realpath "$0")")"
|
||||||
|
cd "$SCRIPT_DIR/.."
|
||||||
|
|
||||||
|
# run basedpyright checks
|
||||||
|
uv --directory api run basedpyright
|
||||||
|
|
@ -14,5 +14,5 @@ uv run --directory api --dev ruff format ./
|
||||||
# run dotenv-linter linter
|
# run dotenv-linter linter
|
||||||
uv run --project api --dev dotenv-linter ./api/.env.example ./web/.env.example
|
uv run --project api --dev dotenv-linter ./api/.env.example ./web/.env.example
|
||||||
|
|
||||||
# run mypy check
|
# run basedpyright check
|
||||||
dev/mypy-check
|
dev/basedpyright-check
|
||||||
|
|
|
||||||
|
|
@ -35,7 +35,6 @@ if $api_modified; then
|
||||||
|
|
||||||
status=${status:-0}
|
status=${status:-0}
|
||||||
|
|
||||||
|
|
||||||
if [ $status -ne 0 ]; then
|
if [ $status -ne 0 ]; then
|
||||||
echo "Ruff linter on api module error, exit code: $status"
|
echo "Ruff linter on api module error, exit code: $status"
|
||||||
echo "Please run 'dev/reformat' to fix the fixable linting errors."
|
echo "Please run 'dev/reformat' to fix the fixable linting errors."
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue