enhancement: introduce Ruff for Python linter for reordering and removing unused imports with automated pre-commit and sytle check (#2366)

api/core/index/base.py

@@ -4,6 +4,7 @@ from abc import ABC, abstractmethod
 from typing import Any, List
 
 from langchain.schema import BaseRetriever, Document
+
 from models.dataset import Dataset
 
 

api/core/index/index.py

@@ -1,10 +1,11 @@
+from flask import current_app
+from langchain.embeddings import OpenAIEmbeddings
+
 from core.embedding.cached_embedding import CacheEmbedding
 from core.index.keyword_table_index.keyword_table_index import KeywordTableConfig, KeywordTableIndex
 from core.index.vector_index.vector_index import VectorIndex
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
-from flask import current_app
-from langchain.embeddings import OpenAIEmbeddings
 from models.dataset import Dataset
 
 

api/core/index/keyword_table_index/jieba_keyword_table_handler.py

@@ -2,9 +2,10 @@ import re
 from typing import Set
 
 import jieba
-from core.index.keyword_table_index.stopwords import STOPWORDS
 from jieba.analyse import default_tfidf
 
+from core.index.keyword_table_index.stopwords import STOPWORDS
+
 
 class JiebaKeywordTableHandler:
 

api/core/index/keyword_table_index/keyword_table_index.py

@@ -2,12 +2,13 @@ import json
 from collections import defaultdict
 from typing import Any, Dict, List, Optional
 
+from langchain.schema import BaseRetriever, Document
+from pydantic import BaseModel, Extra, Field
+
 from core.index.base import BaseIndex
 from core.index.keyword_table_index.jieba_keyword_table_handler import JiebaKeywordTableHandler
 from extensions.ext_database import db
-from langchain.schema import BaseRetriever, Document
 from models.dataset import Dataset, DatasetKeywordTable, DocumentSegment
-from pydantic import BaseModel, Extra, Field
 
 
 class KeywordTableConfig(BaseModel):

api/core/index/vector_index/base.py

@@ -3,14 +3,14 @@ import logging
 from abc import abstractmethod
 from typing import Any, List, cast
 
-from core.index.base import BaseIndex
-from extensions.ext_database import db
 from langchain.embeddings.base import Embeddings
 from langchain.schema import BaseRetriever, Document
 from langchain.vectorstores import VectorStore
-from models.dataset import Dataset, DatasetCollectionBinding
+
+from core.index.base import BaseIndex
+from extensions.ext_database import db
+from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment
 from models.dataset import Document as DatasetDocument
-from models.dataset import DocumentSegment
 
 
 class BaseVectorIndex(BaseIndex):

api/core/index/vector_index/milvus_vector_index.py

@@ -1,13 +1,14 @@
 from typing import Any, List, cast
 
+from langchain.embeddings.base import Embeddings
+from langchain.schema import Document
+from langchain.vectorstores import VectorStore
+from pydantic import BaseModel, root_validator
+
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.milvus_vector_store import MilvusVectorStore
-from langchain.embeddings.base import Embeddings
-from langchain.schema import Document
-from langchain.vectorstores import VectorStore
 from models.dataset import Dataset
-from pydantic import BaseModel, root_validator
 
 
 class MilvusConfig(BaseModel):

api/core/index/vector_index/qdrant_vector_index.py

@@ -2,16 +2,17 @@ import os
 from typing import Any, List, Optional, cast
 
 import qdrant_client
+from langchain.embeddings.base import Embeddings
+from langchain.schema import Document
+from langchain.vectorstores import VectorStore
+from pydantic import BaseModel
+from qdrant_client.http.models import HnswConfigDiff
+
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.qdrant_vector_store import QdrantVectorStore
 from extensions.ext_database import db
-from langchain.embeddings.base import Embeddings
-from langchain.schema import BaseRetriever, Document
-from langchain.vectorstores import VectorStore
 from models.dataset import Dataset, DatasetCollectionBinding
-from pydantic import BaseModel
-from qdrant_client.http.models import HnswConfigDiff
 
 
 class QdrantConfig(BaseModel):

api/core/index/vector_index/vector_index.py

@@ -1,9 +1,10 @@
 import json
 
+from flask import current_app
+from langchain.embeddings.base import Embeddings
+
 from core.index.vector_index.base import BaseVectorIndex
 from extensions.ext_database import db
-from flask import current_app
-from langchain.embeddings.base import Embeddings
 from models.dataset import Dataset, Document
 
 

api/core/index/vector_index/weaviate_vector_index.py

@@ -2,14 +2,15 @@ from typing import Any, List, Optional, cast
 
 import requests
 import weaviate
+from langchain.embeddings.base import Embeddings
+from langchain.schema import Document
+from langchain.vectorstores import VectorStore
+from pydantic import BaseModel, root_validator
+
 from core.index.base import BaseIndex
 from core.index.vector_index.base import BaseVectorIndex
 from core.vector_store.weaviate_vector_store import WeaviateVectorStore
-from langchain.embeddings.base import Embeddings
-from langchain.schema import BaseRetriever, Document
-from langchain.vectorstores import VectorStore
 from models.dataset import Dataset
-from pydantic import BaseModel, root_validator
 
 
 class WeaviateConfig(BaseModel):