jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: decrease claude model request time by set max top_k to 10 (#1071)

Browse source
This commit is contained in:
takatost 2023-08-31 18:23:44 +08:00 • committed by GitHub
commit fcc6b41ab7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
api/core/orchestrator_rule_parser.py
View file

@ -283,6 +283,7 @@ class OrchestratorRuleParser:
def _dynamic_calc_retrieve_k(cls, dataset: Dataset, rest_tokens: int) -> int: def _dynamic_calc_retrieve_k(cls, dataset: Dataset, rest_tokens: int) -> int:
DEFAULT_K = 2 DEFAULT_K = 2
CONTEXT_TOKENS_PERCENT = 0.3 CONTEXT_TOKENS_PERCENT = 0.3
MAX_K = 10
if rest_tokens == -1: if rest_tokens == -1:
return DEFAULT_K return DEFAULT_K
@ -311,5 +312,5 @@ class OrchestratorRuleParser:
if context_limit_tokens <= segment_max_tokens * DEFAULT_K: if context_limit_tokens <= segment_max_tokens * DEFAULT_K:
return DEFAULT_K return DEFAULT_K
# Expand the k value when there's still some room left in the 30% rest tokens space # Expand the k value when there's still some room left in the 30% rest tokens space, but less than the MAX_K
return context_limit_tokens // segment_max_tokens return min(context_limit_tokens // segment_max_tokens, MAX_K)