jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: extract retrival method literal values into enum (#5060)

Browse source
This commit is contained in:
Bowen Liang 2024-06-19 16:05:27 +08:00 • committed by GitHub
commit c923684edd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 47 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

13
api/core/rag/datasource/retrieval_service.py
View file

@ -6,11 +6,12 @@ from flask import Flask, current_app
from core.rag.data_post_processor.data_post_processor import DataPostProcessor
from core.rag.datasource.keyword.keyword_factory import Keyword
from core.rag.datasource.vdb.vector_factory import Vector
from core.rag.retrieval.retrival_methods import RetrievalMethod
from extensions.ext_database import db
from models.dataset import Dataset
default_retrieval_model = {
'search_method': 'semantic_search',
'search_method': RetrievalMethod.SEMANTIC_SEARCH,
'reranking_enable': False,
'reranking_model': {
'reranking_provider_name': '',
@ -47,7 +48,7 @@ class RetrievalService:
threads.append(keyword_thread)
keyword_thread.start()
# retrieval_model source with semantic
if retrival_method == 'semantic_search' or retrival_method == 'hybrid_search':
if RetrievalMethod.is_support_semantic_search(retrival_method):
embedding_thread = threading.Thread(target=RetrievalService.embedding_search, kwargs={
'flask_app': current_app._get_current_object(),
'dataset_id': dataset_id,
@ -63,7 +64,7 @@ class RetrievalService:
embedding_thread.start()
# retrieval source with full text
if retrival_method == 'full_text_search' or retrival_method == 'hybrid_search':
if RetrievalMethod.is_support_fulltext_search(retrival_method):
full_text_index_thread = threading.Thread(target=RetrievalService.full_text_index_search, kwargs={
'flask_app': current_app._get_current_object(),
'dataset_id': dataset_id,
@ -85,7 +86,7 @@ class RetrievalService:
exception_message = ';\n'.join(exceptions)
raise Exception(exception_message)
if retrival_method == 'hybrid_search':
if retrival_method == RetrievalMethod.HYBRID_SEARCH:
data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
all_documents = data_post_processor.invoke(
query=query,
@ -141,7 +142,7 @@ class RetrievalService:
)
if documents:
if reranking_model and retrival_method == 'semantic_search':
if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH:
data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
all_documents.extend(data_post_processor.invoke(
query=query,
@ -173,7 +174,7 @@ class RetrievalService:
top_k=top_k
)
if documents:
if reranking_model and retrival_method == 'full_text_search':
if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH:
data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
all_documents.extend(data_post_processor.invoke(
query=query,

5
api/core/rag/retrieval/dataset_retrieval.py
View file

@ -15,6 +15,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large
from core.rag.datasource.retrieval_service import RetrievalService
from core.rag.models.document import Document
from core.rag.rerank.rerank import RerankRunner
from core.rag.retrieval.retrival_methods import RetrievalMethod
from core.rag.retrieval.router.multi_dataset_function_call_router import FunctionCallMultiDatasetRouter
from core.rag.retrieval.router.multi_dataset_react_route import ReactMultiDatasetRouter
from core.tools.tool.dataset_retriever.dataset_multi_retriever_tool import DatasetMultiRetrieverTool
@ -25,7 +26,7 @@ from models.dataset import Dataset, DatasetQuery, DocumentSegment
from models.dataset import Document as DatasetDocument
default_retrieval_model = {
'search_method': 'semantic_search',
'search_method': RetrievalMethod.SEMANTIC_SEARCH,
'reranking_enable': False,
'reranking_model': {
'reranking_provider_name': '',
@ -419,7 +420,7 @@ class DatasetRetrieval:
if retrieve_config.retrieve_strategy == DatasetRetrieveConfigEntity.RetrieveStrategy.SINGLE:
# get retrieval model config
default_retrieval_model = {
'search_method': 'semantic_search',
'search_method': RetrievalMethod.SEMANTIC_SEARCH,
'reranking_enable': False,
'reranking_model': {
'reranking_provider_name': '',

15
api/core/rag/retrieval/retrival_methods.py Normal file
View file

@ -0,0 +1,15 @@
from enum import Enum
class RetrievalMethod(str, Enum):
SEMANTIC_SEARCH = 'semantic_search'
FULL_TEXT_SEARCH = 'full_text_search'
HYBRID_SEARCH = 'hybrid_search'
@staticmethod
def is_support_semantic_search(retrieval_method: str) -> bool:
return retrieval_method in {RetrievalMethod.SEMANTIC_SEARCH, RetrievalMethod.HYBRID_SEARCH}
@staticmethod
def is_support_fulltext_search(retrieval_method: str) -> bool:
return retrieval_method in {RetrievalMethod.FULL_TEXT_SEARCH, RetrievalMethod.HYBRID_SEARCH}