jyapayne/dify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: optimize xinference stream (#989)

Browse source
This commit is contained in:
takatost 2023-08-24 13:58:34 +08:00 • committed by GitHub
commit 18d3877151
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api/core/third_party/langchain/llms/xinference_llm.py vendored
View file

@ -108,12 +108,12 @@ class XinferenceLLM(Xinference):
Yields: Yields:
A string token. A string token.
""" """
if isinstance(model, RESTfulGenerateModelHandle): if isinstance(model, (RESTfulChatModelHandle, RESTfulChatglmCppChatModelHandle)):
streaming_response = model.generate( streaming_response = model.chat(
prompt=prompt, generate_config=generate_config prompt=prompt, generate_config=generate_config
) )
else: else:
streaming_response = model.chat( streaming_response = model.generate(
prompt=prompt, generate_config=generate_config prompt=prompt, generate_config=generate_config
) )