enhancement: introduce Ruff for Python linter for reordering and removing unused imports with automated pre-commit and sytle check (#2366)

api/app.py

@@ -19,18 +19,28 @@ import threading
 import time
 import warnings
 
-from commands import register_commands
-from config import CloudEditionConfig, Config
-from events import event_handlers
-from extensions import (ext_celery, ext_code_based_extension, ext_database, ext_hosting_provider, ext_login, ext_mail,
-                        ext_migrate, ext_redis, ext_sentry, ext_storage)
-from extensions.ext_database import db
-from extensions.ext_login import login_manager
 from flask import Flask, Response, request
 from flask_cors import CORS
+
+from commands import register_commands
+from config import CloudEditionConfig, Config
+from extensions import (
+    ext_celery,
+    ext_code_based_extension,
+    ext_database,
+    ext_hosting_provider,
+    ext_login,
+    ext_mail,
+    ext_migrate,
+    ext_redis,
+    ext_sentry,
+    ext_storage,
+)
+from extensions.ext_database import db
+from extensions.ext_login import login_manager
 from libs.passport import PassportService
+
 # DO NOT REMOVE BELOW
-from models import account, dataset, model, source, task, tool, tools, web
 from services.account_service import AccountService
 
 # DO NOT REMOVE ABOVE

api/commands.py

@@ -3,11 +3,13 @@ import json
 import secrets
 
 import click
+from flask import current_app
+from werkzeug.exceptions import NotFound
+
 from core.embedding.cached_embedding import CacheEmbedding
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from extensions.ext_database import db
-from flask import current_app
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -15,7 +17,6 @@ from models.account import Tenant
 from models.dataset import Dataset
 from models.model import Account
 from models.provider import Provider, ProviderModel
-from werkzeug.exceptions import NotFound
 
 
 @click.command('reset-password', help='Reset the account password.')

api/constants/model_template.py

@@ -1,7 +1,5 @@
 import json
 
-from models.model import App, AppModelConfig
-
 model_templates = {
     # completion default mode
     'completion_default': {

api/controllers/console/admin.py

@@ -1,14 +1,15 @@
 import os
 from functools import wraps
 
+from flask import request
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import NotFound, Unauthorized
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.wraps import only_edition_cloud
 from extensions.ext_database import db
-from flask import request
-from flask_restful import Resource, reqparse
 from models.model import App, InstalledApp, RecommendedApp
-from werkzeug.exceptions import NotFound, Unauthorized
 
 
 def admin_required(view):

api/controllers/console/apikey.py

@@ -1,12 +1,13 @@
 import flask_restful
-from extensions.ext_database import db
 from flask_login import current_user
 from flask_restful import Resource, fields, marshal_with
+from werkzeug.exceptions import Forbidden
+
+from extensions.ext_database import db
 from libs.helper import TimestampField
 from libs.login import login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
-from werkzeug.exceptions import Forbidden
 
 from . import api
 from .setup import setup_required

api/controllers/console/app/advanced_prompt_template.py

@@ -1,7 +1,8 @@
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 from services.advanced_prompt_template_service import AdvancedPromptTemplateService
 

api/controllers/console/app/annotation.py

@@ -1,17 +1,20 @@
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource, marshal, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from controllers.console import api
 from controllers.console.app.error import NoFileUploadedError
 from controllers.console.datasets.error import TooManyFilesError
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import (annotation_fields, annotation_hit_history_fields,
-                                      annotation_hit_history_list_fields, annotation_list_fields)
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource, marshal, marshal_with, reqparse
+from fields.annotation_fields import (
+    annotation_fields,
+    annotation_hit_history_fields,
+)
 from libs.login import login_required
 from services.annotation_service import AppAnnotationService
-from werkzeug.exceptions import Forbidden
 
 
 class AnnotationReplyActionApi(Resource):

api/controllers/console/app/app.py

@@ -3,6 +3,10 @@ import json
 import logging
 from datetime import datetime
 
+from flask_login import current_user
+from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden
+
 from constants.languages import demo_model_templates, languages
 from constants.model_template import model_templates
 from controllers.console import api
@@ -15,16 +19,15 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from events.app_event import app_was_created, app_was_deleted
 from extensions.ext_database import db
-from fields.app_fields import (app_detail_fields, app_detail_fields_with_site, app_pagination_fields,
-                               template_list_fields)
-from flask import current_app
-from flask_login import current_user
-from flask_restful import Resource, abort, inputs, marshal_with, reqparse
+from fields.app_fields import (
+    app_detail_fields,
+    app_detail_fields_with_site,
+    app_pagination_fields,
+    template_list_fields,
+)
 from libs.login import login_required
 from models.model import App, AppModelConfig, Site
-from models.tools import ApiToolProvider
 from services.app_model_config_service import AppModelConfigService
-from werkzeug.exceptions import Forbidden
 
 
 def _get_app(app_id, tenant_id):

api/controllers/console/app/audio.py

@@ -1,24 +1,36 @@
 # -*- coding:utf-8 -*-
 import logging
 
+from flask import request
+from flask_restful import Resource
+from werkzeug.exceptions import InternalServerError
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, AudioTooLargeError, CompletionRequestError,
-                                           NoAudioUploadedError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderNotSupportSpeechToTextError,
-                                           ProviderQuotaExceededError, UnsupportedAudioTypeError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    AudioTooLargeError,
+    CompletionRequestError,
+    NoAudioUploadedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderNotSupportSpeechToTextError,
+    ProviderQuotaExceededError,
+    UnsupportedAudioTypeError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import request
-from flask_restful import Resource
 from libs.login import login_required
 from services.audio_service import AudioService
-from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError,
-                                   ProviderNotSupportSpeechToTextServiceError, UnsupportedAudioTypeServiceError)
-from werkzeug.exceptions import InternalServerError
+from services.errors.audio import (
+    AudioTooLargeServiceError,
+    NoAudioUploadedServiceError,
+    ProviderNotSupportSpeechToTextServiceError,
+    UnsupportedAudioTypeServiceError,
+)
 
 
 class ChatMessageAudioApi(Resource):

api/controllers/console/app/completion.py

@@ -4,24 +4,30 @@ import logging
 from typing import Generator, Union
 
 import flask_login
+from flask import Response, stream_with_context
+from flask_restful import Resource, reqparse
+from werkzeug.exceptions import InternalServerError, NotFound
+
 import services
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppUnavailableError, CompletionRequestError, ConversationCompletedError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppUnavailableError,
+    CompletionRequestError,
+    ConversationCompletedError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.application_queue_manager import ApplicationQueueManager
 from core.entities.application_entities import InvokeFrom
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
-from flask import Response, stream_with_context
-from flask_restful import Resource, reqparse
 from libs.helper import uuid_value
 from libs.login import login_required
 from services.completion_service import CompletionService
-from werkzeug.exceptions import InternalServerError, NotFound
 
 
 # define completion message api for user

api/controllers/console/app/conversation.py

@@ -1,22 +1,27 @@
 from datetime import datetime
 
 import pytz
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from sqlalchemy import func, or_
+from sqlalchemy.orm import joinedload
+from werkzeug.exceptions import NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from fields.conversation_fields import (conversation_detail_fields, conversation_message_detail_fields,
-                                        conversation_pagination_fields, conversation_with_summary_pagination_fields)
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
-from flask_restful.inputs import int_range
+from fields.conversation_fields import (
+    conversation_detail_fields,
+    conversation_message_detail_fields,
+    conversation_pagination_fields,
+    conversation_with_summary_pagination_fields,
+)
 from libs.helper import datetime_string
 from libs.login import login_required
 from models.model import Conversation, Message, MessageAnnotation
-from sqlalchemy import func, or_
-from sqlalchemy.orm import joinedload
-from werkzeug.exceptions import NotFound
 
 
 class CompletionConversationApi(Resource):

api/controllers/console/app/generator.py

@@ -1,13 +1,18 @@
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
-from controllers.console.app.error import (CompletionRequestError, ProviderModelCurrentlyNotSupportError,
-                                           ProviderNotInitializeError, ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.generator.llm_generator import LLMGenerator
 from core.model_runtime.errors.invoke import InvokeError
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.login import login_required
 
 

api/controllers/console/app/message.py

@@ -2,11 +2,21 @@ import json
 import logging
 from typing import Generator, Union
 
+from flask import Response, stream_with_context
+from flask_login import current_user
+from flask_restful import Resource, fields, marshal_with, reqparse
+from flask_restful.inputs import int_range
+from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
+
 from controllers.console import api
 from controllers.console.app import _get_app
-from controllers.console.app.error import (AppMoreLikeThisDisabledError, CompletionRequestError,
-                                           ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError,
-                                           ProviderQuotaExceededError)
+from controllers.console.app.error import (
+    AppMoreLikeThisDisabledError,
+    CompletionRequestError,
+    ProviderModelCurrentlyNotSupportError,
+    ProviderNotInitializeError,
+    ProviderQuotaExceededError,
+)
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check
 from core.entities.application_entities import InvokeFrom
@@ -14,10 +24,6 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
 from core.model_runtime.errors.invoke import InvokeError
 from extensions.ext_database import db
 from fields.conversation_fields import annotation_fields, message_detail_fields
-from flask import Response, stream_with_context
-from flask_login import current_user
-from flask_restful import Resource, fields, marshal_with, reqparse
-from flask_restful.inputs import int_range
 from libs.helper import uuid_value
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from libs.login import login_required
@@ -28,7 +34,6 @@ from services.errors.app import MoreLikeThisDisabledError
 from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError
 from services.message_service import MessageService
-from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 
 class ChatMessageListApi(Resource):

api/controllers/console/app/model_config.py

@@ -1,14 +1,15 @@
 # -*- coding:utf-8 -*-
 
+from flask import request
+from flask_login import current_user
+from flask_restful import Resource
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from events.app_event import app_model_config_was_updated
 from extensions.ext_database import db
-from flask import request
-from flask_login import current_user
-from flask_restful import Resource
 from libs.login import login_required
 from models.model import AppModelConfig
 from services.app_model_config_service import AppModelConfigService

api/controllers/console/app/site.py

@@ -1,4 +1,8 @@
 # -*- coding:utf-8 -*-
+from flask_login import current_user
+from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import Forbidden, NotFound
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.app import _get_app
@@ -6,11 +10,8 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
 from fields.app_fields import app_site_fields
-from flask_login import current_user
-from flask_restful import Resource, marshal_with, reqparse
 from libs.login import login_required
 from models.model import Site
-from werkzeug.exceptions import Forbidden, NotFound
 
 
 def parse_app_site_args():

api/controllers/console/app/statistic.py

@@ -3,14 +3,15 @@ from datetime import datetime
 from decimal import Decimal
 
 import pytz
+from flask import jsonify
+from flask_login import current_user
+from flask_restful import Resource, reqparse
+
 from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
-from flask import jsonify
-from flask_login import current_user
-from flask_restful import Resource, reqparse
 from libs.helper import datetime_string
 from libs.login import login_required
 

api/controllers/console/auth/activate.py

@@ -2,14 +2,15 @@ import base64
 import secrets
 from datetime import datetime
 
+from flask_restful import Resource, reqparse
+
 from constants.languages import supported_language
 from controllers.console import api
 from controllers.console.error import AlreadyActivateError
 from extensions.ext_database import db
-from flask_restful import Resource, reqparse
 from libs.helper import email, str_len, timezone
 from libs.password import hash_password, valid_password
-from models.account import AccountStatus, Tenant
+from models.account import AccountStatus
 from services.account_service import RegisterService
 
 

api/controllers/console/auth/data_source_oauth.py

@@ -1,13 +1,14 @@
 import logging
 
 import requests
-from controllers.console import api
 from flask import current_app, redirect, request
 from flask_login import current_user
 from flask_restful import Resource
+from werkzeug.exceptions import Forbidden
+
+from controllers.console import api
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
-from werkzeug.exceptions import Forbidden
 
 from ..setup import setup_required
 from ..wraps import account_initialization_required

api/controllers/console/auth/login.py

@@ -1,11 +1,11 @@
 # -*- coding:utf-8 -*-
-import flask
 import flask_login
+from flask import current_app, request
+from flask_restful import Resource, reqparse
+
 import services
 from controllers.console import api
 from controllers.console.setup import setup_required
-from flask import current_app, request
-from flask_restful import Resource, reqparse
 from libs.helper import email
 from libs.password import valid_password
 from services.account_service import AccountService

api/controllers/console/auth/oauth.py

@@ -3,10 +3,11 @@ from datetime import datetime
 from typing import Optional
 
 import requests
-from constants.languages import languages
-from extensions.ext_database import db
 from flask import current_app, redirect, request
 from flask_restful import Resource
+
+from constants.languages import languages
+from extensions.ext_database import db
 from libs.oauth import GitHubOAuth, GoogleOAuth, OAuthUserInfo
 from models.account import Account, AccountStatus
 from services.account_service import AccountService, RegisterService