jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix: Wrong cutoff length lead to empty input in openai compatible embedding model. (#7133)

Browse source
This commit is contained in:
Yanyi Liu 2024-08-09 19:11:57 +08:00 • committed by GitHub
commit 4cbeb6815b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py
View file

@ -72,7 +72,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens)))
cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff])
else:

2
api/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py
View file

@ -76,7 +76,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens)))
cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff])
else:

2
api/core/model_runtime/model_providers/perfxcloud/text_embedding/text_embedding.py
View file

@ -79,7 +79,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel):
num_tokens = self._get_num_tokens_by_gpt2(text)
if num_tokens >= context_size:
cutoff = int(len(text) * (np.floor(context_size / num_tokens)))
cutoff = int(np.floor(len(text) * (context_size / num_tokens)))
# if num tokens is larger than context length, only use the start
inputs.append(text[0: cutoff])
else: