Model Runtime (#1858)

Co-authored-by: StyleZhang <jasonapring2015@outlook.com>
Co-authored-by: Garfield Dai <dai.hai@foxmail.com>
Co-authored-by: chenhe <guchenhe@gmail.com>
Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: Joel <iamjoel007@gmail.com>
Co-authored-by: Yeuoly <admin@srmxy.cn>

api/core/model_runtime/model_providers/__base/ai_model.py

@@ -0,0 +1,328 @@
+import decimal
+import json
+import logging
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+
+import yaml
+from pydantic import ValidationError
+
+from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
+from core.model_runtime.entities.model_entities import PriceInfo, AIModelEntity, PriceType, PriceConfig, \
+    DefaultParameterName, FetchFrom, ModelType
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError
+from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
+
+
+class AIModel(ABC):
+    """
+    Base class for all models.
+    """
+    model_type: ModelType
+    model_schemas: list[AIModelEntity] = None
+    started_at: float = 0
+
+    @abstractmethod
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        raise NotImplementedError
+
+    def _transform_invoke_error(self, error: Exception) -> InvokeError:
+        """
+        Transform invoke error to unified error
+
+        :param error: model invoke error
+        :return: unified error
+        """
+        for invoke_error, model_errors in self._invoke_error_mapping.items():
+            if isinstance(error, tuple(model_errors)):
+                if invoke_error == InvokeAuthorizationError:
+                    return invoke_error(description="Incorrect model credentials provided, please check and try again. ")
+
+                return invoke_error(description=f"{invoke_error.description}: {str(error)}")
+
+        return InvokeError(description=f"Error: {str(error)}")
+
+    def get_price(self, model: str, credentials: dict, price_type: PriceType, tokens: int) -> PriceInfo:
+        """
+        Get price for given model and tokens
+
+        :param model: model name
+        :param credentials: model credentials
+        :param price_type: price type
+        :param tokens: number of tokens
+        :return: price info
+        """
+        # get model schema
+        model_schema = self.get_model_schema(model, credentials)
+
+        # get price info from predefined model schema
+        price_config: Optional[PriceConfig] = None
+        if model_schema:
+            price_config: PriceConfig = model_schema.pricing
+
+        # get unit price
+        unit_price = None
+        if price_config:
+            if price_type == PriceType.INPUT:
+                unit_price = price_config.input
+            elif price_type == PriceType.OUTPUT and price_config.output is not None:
+                unit_price = price_config.output
+
+        if unit_price is None:
+            return PriceInfo(
+                unit_price=decimal.Decimal('0.0'),
+                unit=decimal.Decimal('0.0'),
+                total_amount=decimal.Decimal('0.0'),
+                currency="USD",
+            )
+
+        # calculate total amount
+        total_amount = tokens * unit_price * price_config.unit
+        total_amount = total_amount.quantize(decimal.Decimal('0.0000001'), rounding=decimal.ROUND_HALF_UP)
+
+        return PriceInfo(
+            unit_price=unit_price,
+            unit=price_config.unit,
+            total_amount=total_amount,
+            currency=price_config.currency,
+        )
+
+    def predefined_models(self) -> list[AIModelEntity]:
+        """
+        Get all predefined models for given provider.
+
+        :return:
+        """
+        if self.model_schemas:
+            return self.model_schemas
+
+        model_schemas = []
+
+        # get module name
+        model_type = self.__class__.__module__.split('.')[-1]
+
+        # get provider name
+        provider_name = self.__class__.__module__.split('.')[-3]
+
+        # get the path of current classes
+        current_path = os.path.abspath(__file__)
+        # get parent path of the current path
+        provider_model_type_path = os.path.join(os.path.dirname(os.path.dirname(current_path)), provider_name, model_type)
+
+        # get all yaml files path under provider_model_type_path that do not start with __
+        model_schema_yaml_paths = [
+            os.path.join(provider_model_type_path, model_schema_yaml)
+            for model_schema_yaml in os.listdir(provider_model_type_path)
+            if not model_schema_yaml.startswith('__')
+               and not model_schema_yaml.startswith('_')
+               and os.path.isfile(os.path.join(provider_model_type_path, model_schema_yaml))
+               and model_schema_yaml.endswith('.yaml')
+        ]
+
+        # get _position.yaml file path
+        position_file_path = os.path.join(provider_model_type_path, '_position.yaml')
+
+        # read _position.yaml file
+        position_map = {}
+        if os.path.exists(position_file_path):
+            with open(position_file_path, 'r') as f:
+                position_map = yaml.safe_load(f)
+
+        # traverse all model_schema_yaml_paths
+        for model_schema_yaml_path in model_schema_yaml_paths:
+            # read yaml data from yaml file
+            with open(model_schema_yaml_path, 'r') as f:
+                yaml_data = yaml.safe_load(f)
+
+            new_parameter_rules = []
+            for parameter_rule in yaml_data.get('parameter_rules', []):
+                if 'use_template' in parameter_rule:
+                    try:
+                        default_parameter_name = DefaultParameterName.value_of(parameter_rule['use_template'])
+                        default_parameter_rule = self._get_default_parameter_rule_variable_map(default_parameter_name)
+                        copy_default_parameter_rule = default_parameter_rule.copy()
+                        copy_default_parameter_rule.update(parameter_rule)
+                        parameter_rule = copy_default_parameter_rule
+                    except ValueError:
+                        pass
+
+                if 'label' not in parameter_rule:
+                    parameter_rule['label'] = {
+                        'zh_Hans': parameter_rule['name'],
+                        'en_US': parameter_rule['name']
+                    }
+
+                new_parameter_rules.append(parameter_rule)
+
+            yaml_data['parameter_rules'] = new_parameter_rules
+
+            if 'label' not in yaml_data:
+                yaml_data['label'] = {
+                    'zh_Hans': yaml_data['model'],
+                    'en_US': yaml_data['model']
+                }
+
+            yaml_data['fetch_from'] = FetchFrom.PREDEFINED_MODEL.value
+
+            try:
+                # yaml_data to entity
+                model_schema = AIModelEntity(**yaml_data)
+            except Exception as e:
+                model_schema_yaml_file_name = os.path.basename(model_schema_yaml_path).rstrip(".yaml")
+                raise Exception(f'Invalid model schema for {provider_name}.{model_type}.{model_schema_yaml_file_name}:'
+                                f' {str(e)}')
+
+            # cache model schema
+            model_schemas.append(model_schema)
+
+        # resort model schemas by position
+        if position_map:
+            model_schemas.sort(key=lambda x: position_map.get(x.model, 999))
+
+        # cache model schemas
+        self.model_schemas = model_schemas
+
+        return model_schemas
+
+    def get_model_schema(self, model: str, credentials: Optional[dict] = None) -> Optional[AIModelEntity]:
+        """
+        Get model schema by model name and credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: model schema
+        """
+        # get predefined models (predefined_models)
+        models = self.predefined_models()
+
+        model_map = {model.model: model for model in models}
+        if model in model_map:
+            return model_map[model]
+
+        if credentials:
+            model_schema = self.get_customizable_model_schema_from_credentials(model, credentials)
+            if model_schema:
+                return model_schema
+
+        return None
+
+    def get_customizable_model_schema_from_credentials(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        """
+        Get customizable model schema from credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: model schema
+        """
+        if 'schema' in credentials:
+            schema_dict = json.loads(credentials['schema'])
+
+            try:
+                model_instance = AIModelEntity.parse_obj(schema_dict)
+                return model_instance
+            except ValidationError as e:
+                logging.exception(f"Invalid model schema for {model}")
+                return self._get_customizable_model_schema(model, credentials)
+
+        return self._get_customizable_model_schema(model, credentials)
+    
+    def _get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        """
+        Get customizable model schema and fill in the template
+        """
+        schema = self.get_customizable_model_schema(model, credentials)
+
+        if not schema:
+            return None
+        
+        # fill in the template
+        new_parameter_rules = []
+        for parameter_rule in schema.parameter_rules:
+            if parameter_rule.use_template:
+                try:
+                    default_parameter_name = DefaultParameterName.value_of(parameter_rule.use_template)
+                    default_parameter_rule = self._get_default_parameter_rule_variable_map(default_parameter_name)
+                    if not parameter_rule.max:
+                        parameter_rule.max = default_parameter_rule['max']
+                    if not parameter_rule.min:
+                        parameter_rule.min = default_parameter_rule['min']
+                    if not parameter_rule.precision:
+                        parameter_rule.default = default_parameter_rule['default']
+                    if not parameter_rule.precision:
+                        parameter_rule.precision = default_parameter_rule['precision']
+                    if not parameter_rule.required:
+                        parameter_rule.required = default_parameter_rule['required']
+                    if not parameter_rule.help:
+                        parameter_rule.help = I18nObject(
+                            en_US=default_parameter_rule['help']['en_US'],
+                        )
+                    if not parameter_rule.help.en_US:
+                        parameter_rule.help.en_US = default_parameter_rule['help']['en_US']
+                    if not parameter_rule.help.zh_Hans:
+                        parameter_rule.help.zh_Hans = default_parameter_rule['help'].get('zh_Hans', default_parameter_rule['help']['en_US'])
+                except ValueError:
+                    pass
+
+            new_parameter_rules.append(parameter_rule)
+
+        schema.parameter_rules = new_parameter_rules
+
+        return schema
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        """
+        Get customizable model schema
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: model schema
+        """
+        return None
+
+    def _get_default_parameter_rule_variable_map(self, name: DefaultParameterName) -> dict:
+        """
+        Get default parameter rule for given name
+
+        :param name: parameter name
+        :return: parameter rule
+        """
+        default_parameter_rule = PARAMETER_RULE_TEMPLATE.get(name)
+
+        if not default_parameter_rule:
+            raise Exception(f'Invalid model parameter rule name {name}')
+
+        return default_parameter_rule
+
+    def _get_num_tokens_by_gpt2(self, text: str) -> int:
+        """
+        Get number of tokens for given prompt messages by gpt2
+        Some provider models do not provide an interface for obtaining the number of tokens.
+        Here, the gpt2 tokenizer is used to calculate the number of tokens.
+        This method can be executed offline, and the gpt2 tokenizer has been cached in the project.
+
+        :param text: plain text of prompt. You need to convert the original message to plain text
+        :return: number of tokens
+        """
+        return GPT2Tokenizer.get_num_tokens(text)

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -0,0 +1,557 @@
+import logging
+import os
+import time
+from abc import abstractmethod
+from typing import Optional, Generator, Union, List
+
+from core.model_runtime.callbacks.base_callback import Callback
+from core.model_runtime.callbacks.logging_callback import LoggingCallback
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage
+from core.model_runtime.entities.model_entities import ModelPropertyKey, PriceType, ParameterType, ParameterRule, \
+    ModelType
+from core.model_runtime.entities.llm_entities import LLMResult, LLMMode, LLMUsage, \
+    LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+logger = logging.getLogger(__name__)
+
+
+class LargeLanguageModel(AIModel):
+    """
+    Model class for large language model.
+    """
+    model_type: ModelType = ModelType.LLM
+
+    def invoke(self, model: str, credentials: dict,
+               prompt_messages: list[PromptMessage], model_parameters: Optional[dict] = None,
+               tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+               stream: bool = True, user: Optional[str] = None, callbacks: list[Callback] = None) \
+            -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :param callbacks: callbacks
+        :return: full response or stream response chunk generator result
+        """
+        # validate and filter model parameters
+        if model_parameters is None:
+            model_parameters = {}
+
+        model_parameters = self._validate_and_filter_model_parameters(model, model_parameters, credentials)
+
+        self.started_at = time.perf_counter()
+
+        callbacks = callbacks or []
+
+        if bool(os.environ.get("DEBUG")):
+            callbacks.append(LoggingCallback())
+
+        # trigger before invoke callbacks
+        self._trigger_before_invoke_callbacks(
+            model=model,
+            credentials=credentials,
+            prompt_messages=prompt_messages,
+            model_parameters=model_parameters,
+            tools=tools,
+            stop=stop,
+            stream=stream,
+            user=user,
+            callbacks=callbacks
+        )
+
+        try:
+            result = self._invoke(model, credentials, prompt_messages, model_parameters, tools, stop, stream, user)
+        except Exception as e:
+            self._trigger_invoke_error_callbacks(
+                model=model,
+                ex=e,
+                credentials=credentials,
+                prompt_messages=prompt_messages,
+                model_parameters=model_parameters,
+                tools=tools,
+                stop=stop,
+                stream=stream,
+                user=user,
+                callbacks=callbacks
+            )
+
+            raise self._transform_invoke_error(e)
+
+        if stream and isinstance(result, Generator):
+            return self._invoke_result_generator(
+                model=model,
+                result=result,
+                credentials=credentials,
+                prompt_messages=prompt_messages,
+                model_parameters=model_parameters,
+                tools=tools,
+                stop=stop,
+                stream=stream,
+                user=user,
+                callbacks=callbacks
+            )
+        else:
+            self._trigger_after_invoke_callbacks(
+                model=model,
+                result=result,
+                credentials=credentials,
+                prompt_messages=prompt_messages,
+                model_parameters=model_parameters,
+                tools=tools,
+                stop=stop,
+                stream=stream,
+                user=user,
+                callbacks=callbacks
+            )
+
+        return result
+
+    def _invoke_result_generator(self, model: str, result: Generator, credentials: dict,
+                                 prompt_messages: list[PromptMessage], model_parameters: dict,
+                                 tools: Optional[list[PromptMessageTool]] = None,
+                                 stop: Optional[List[str]] = None, stream: bool = True,
+                                 user: Optional[str] = None, callbacks: list[Callback] = None) -> Generator:
+        """
+        Invoke result generator
+
+        :param result: result generator
+        :return: result generator
+        """
+        prompt_message = AssistantPromptMessage(
+            content=""
+        )
+        usage = None
+        system_fingerprint = None
+        real_model = model
+
+        for chunk in result:
+            try:
+                yield chunk
+
+                self._trigger_new_chunk_callbacks(
+                    chunk=chunk,
+                    model=model,
+                    credentials=credentials,
+                    prompt_messages=prompt_messages,
+                    model_parameters=model_parameters,
+                    tools=tools,
+                    stop=stop,
+                    stream=stream,
+                    user=user,
+                    callbacks=callbacks
+                )
+
+                prompt_message.content += chunk.delta.message.content
+                real_model = chunk.model
+                if chunk.delta.usage:
+                    usage = chunk.delta.usage
+
+                if chunk.system_fingerprint:
+                    system_fingerprint = chunk.system_fingerprint
+            except Exception as e:
+                raise self._transform_invoke_error(e)
+
+        self._trigger_after_invoke_callbacks(
+            model=model,
+            result=LLMResult(
+                model=real_model,
+                prompt_messages=prompt_messages,
+                message=prompt_message,
+                usage=usage,
+                system_fingerprint=system_fingerprint
+            ),
+            credentials=credentials,
+            prompt_messages=prompt_messages,
+            model_parameters=model_parameters,
+            tools=tools,
+            stop=stop,
+            stream=stream,
+            user=user,
+            callbacks=callbacks
+        )
+
+    @abstractmethod
+    def _invoke(self, model: str, credentials: dict,
+                prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+                stream: bool = True, user: Optional[str] = None) \
+            -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param tools: tools for tool calling
+        :return:
+        """
+        raise NotImplementedError
+
+    def _llm_result_to_stream(self, result: LLMResult) -> Generator:
+        """
+        Transform llm result to stream
+
+        :param result: llm result
+        :return: stream
+        """
+        index = 0
+
+        tool_calls = result.message.tool_calls
+
+        for word in result.message.content:
+            assistant_prompt_message = AssistantPromptMessage(
+                content=word,
+                tool_calls=tool_calls if index == (len(result.message.content) - 1) else []
+            )
+
+            yield LLMResultChunk(
+                model=result.model,
+                prompt_messages=result.prompt_messages,
+                system_fingerprint=result.system_fingerprint,
+                delta=LLMResultChunkDelta(
+                    index=index,
+                    message=assistant_prompt_message,
+                )
+            )
+
+            index += 1
+            time.sleep(0.01)
+
+    def get_parameter_rules(self, model: str, credentials: dict) -> list[ParameterRule]:
+        """
+        Get parameter rules
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: parameter rules
+        """
+        model_schema = self.get_model_schema(model, credentials)
+        if model_schema:
+            return model_schema.parameter_rules
+
+        return []
+
+    def get_model_mode(self, model: str, credentials: Optional[dict] = None) -> LLMMode:
+        """
+        Get model mode
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: model mode
+        """
+        model_schema = self.get_model_schema(model, credentials)
+
+        mode = LLMMode.CHAT
+        if model_schema and model_schema.model_properties.get(ModelPropertyKey.MODE):
+            mode = LLMMode.value_of(model_schema.model_properties[ModelPropertyKey.MODE])
+
+        return mode
+
+    def _calc_response_usage(self, model: str, credentials: dict, prompt_tokens: int, completion_tokens: int) -> LLMUsage:
+        """
+        Calculate response usage
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_tokens: prompt tokens
+        :param completion_tokens: completion tokens
+        :return: usage
+        """
+        # get prompt price info
+        prompt_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=prompt_tokens,
+        )
+
+        # get completion price info
+        completion_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.OUTPUT,
+            tokens=completion_tokens
+        )
+
+        # transform usage
+        usage = LLMUsage(
+            prompt_tokens=prompt_tokens,
+            prompt_unit_price=prompt_price_info.unit_price,
+            prompt_price_unit=prompt_price_info.unit,
+            prompt_price=prompt_price_info.total_amount,
+            completion_tokens=completion_tokens,
+            completion_unit_price=completion_price_info.unit_price,
+            completion_price_unit=completion_price_info.unit,
+            completion_price=completion_price_info.total_amount,
+            total_tokens=prompt_tokens + completion_tokens,
+            total_price=prompt_price_info.total_amount + completion_price_info.total_amount,
+            currency=prompt_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage
+
+    def _trigger_before_invoke_callbacks(self, model: str, credentials: dict,
+                                         prompt_messages: list[PromptMessage], model_parameters: dict,
+                                         tools: Optional[list[PromptMessageTool]] = None,
+                                         stop: Optional[List[str]] = None, stream: bool = True,
+                                         user: Optional[str] = None, callbacks: list[Callback] = None) -> None:
+        """
+        Trigger before invoke callbacks
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :param callbacks: callbacks
+        """
+        if callbacks:
+            for callback in callbacks:
+                try:
+                    callback.on_before_invoke(
+                        llm_instance=self,
+                        model=model,
+                        credentials=credentials,
+                        prompt_messages=prompt_messages,
+                        model_parameters=model_parameters,
+                        tools=tools,
+                        stop=stop,
+                        stream=stream,
+                        user=user
+                    )
+                except Exception as e:
+                    if callback.raise_error:
+                        raise e
+                    else:
+                        logger.warning(f"Callback {callback.__class__.__name__} on_before_invoke failed with error {e}")
+
+    def _trigger_new_chunk_callbacks(self, chunk: LLMResultChunk, model: str, credentials: dict,
+                                     prompt_messages: list[PromptMessage], model_parameters: dict,
+                                     tools: Optional[list[PromptMessageTool]] = None,
+                                     stop: Optional[List[str]] = None, stream: bool = True,
+                                     user: Optional[str] = None, callbacks: list[Callback] = None) -> None:
+        """
+        Trigger new chunk callbacks
+
+        :param chunk: chunk
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        """
+        if callbacks:
+            for callback in callbacks:
+                try:
+                    callback.on_new_chunk(
+                        llm_instance=self,
+                        chunk=chunk,
+                        model=model,
+                        credentials=credentials,
+                        prompt_messages=prompt_messages,
+                        model_parameters=model_parameters,
+                        tools=tools,
+                        stop=stop,
+                        stream=stream,
+                        user=user
+                    )
+                except Exception as e:
+                    if callback.raise_error:
+                        raise e
+                    else:
+                        logger.warning(f"Callback {callback.__class__.__name__} on_new_chunk failed with error {e}")
+
+    def _trigger_after_invoke_callbacks(self, model: str, result: LLMResult, credentials: dict,
+                                        prompt_messages: list[PromptMessage], model_parameters: dict,
+                                        tools: Optional[list[PromptMessageTool]] = None,
+                                        stop: Optional[List[str]] = None, stream: bool = True,
+                                        user: Optional[str] = None, callbacks: list[Callback] = None) -> None:
+        """
+        Trigger after invoke callbacks
+
+        :param model: model name
+        :param result: result
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :param callbacks: callbacks
+        """
+        if callbacks:
+            for callback in callbacks:
+                try:
+                    callback.on_after_invoke(
+                        llm_instance=self,
+                        result=result,
+                        model=model,
+                        credentials=credentials,
+                        prompt_messages=prompt_messages,
+                        model_parameters=model_parameters,
+                        tools=tools,
+                        stop=stop,
+                        stream=stream,
+                        user=user
+                    )
+                except Exception as e:
+                    if callback.raise_error:
+                        raise e
+                    else:
+                        logger.warning(f"Callback {callback.__class__.__name__} on_after_invoke failed with error {e}")
+
+    def _trigger_invoke_error_callbacks(self, model: str, ex: Exception, credentials: dict,
+                                        prompt_messages: list[PromptMessage], model_parameters: dict,
+                                        tools: Optional[list[PromptMessageTool]] = None,
+                                        stop: Optional[List[str]] = None, stream: bool = True,
+                                        user: Optional[str] = None, callbacks: list[Callback] = None) -> None:
+        """
+        Trigger invoke error callbacks
+
+        :param model: model name
+        :param ex: exception
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :param callbacks: callbacks
+        """
+        if callbacks:
+            for callback in callbacks:
+                try:
+                    callback.on_invoke_error(
+                        llm_instance=self,
+                        ex=ex,
+                        model=model,
+                        credentials=credentials,
+                        prompt_messages=prompt_messages,
+                        model_parameters=model_parameters,
+                        tools=tools,
+                        stop=stop,
+                        stream=stream,
+                        user=user
+                    )
+                except Exception as e:
+                    if callback.raise_error:
+                        raise e
+                    else:
+                        logger.warning(f"Callback {callback.__class__.__name__} on_invoke_error failed with error {e}")
+
+    def _validate_and_filter_model_parameters(self, model: str, model_parameters: dict, credentials: dict) -> dict:
+        """
+        Validate model parameters
+
+        :param model: model name
+        :param model_parameters: model parameters
+        :param credentials: model credentials
+        :return:
+        """
+        parameter_rules = self.get_parameter_rules(model, credentials)
+
+        # validate model parameters
+        filtered_model_parameters = {}
+        for parameter_rule in parameter_rules:
+            parameter_name = parameter_rule.name
+            parameter_value = model_parameters.get(parameter_name)
+            if parameter_value is None:
+                if parameter_rule.use_template and parameter_rule.use_template in model_parameters:
+                    # if parameter value is None, use template value variable name instead
+                    parameter_value = model_parameters[parameter_rule.use_template]
+                else:
+                    if parameter_rule.required:
+                        if parameter_rule.default is not None:
+                            filtered_model_parameters[parameter_name] = parameter_rule.default
+                            continue
+                        else:
+                            raise ValueError(f"Model Parameter {parameter_name} is required.")
+                    else:
+                        continue
+
+            # validate parameter value type
+            if parameter_rule.type == ParameterType.INT:
+                if not isinstance(parameter_value, int):
+                    raise ValueError(f"Model Parameter {parameter_name} should be int.")
+
+                # validate parameter value range
+                if parameter_rule.min is not None and parameter_value < parameter_rule.min:
+                    raise ValueError(
+                        f"Model Parameter {parameter_name} should be greater than or equal to {parameter_rule.min}.")
+
+                if parameter_rule.max is not None and parameter_value > parameter_rule.max:
+                    raise ValueError(
+                        f"Model Parameter {parameter_name} should be less than or equal to {parameter_rule.max}.")
+            elif parameter_rule.type == ParameterType.FLOAT:
+                if not isinstance(parameter_value, (float, int)):
+                    raise ValueError(f"Model Parameter {parameter_name} should be float.")
+
+                # validate parameter value precision
+                if parameter_rule.precision is not None:
+                    if parameter_rule.precision == 0:
+                        if parameter_value != int(parameter_value):
+                            raise ValueError(f"Model Parameter {parameter_name} should be int.")
+                    else:
+                        if parameter_value != round(parameter_value, parameter_rule.precision):
+                            raise ValueError(
+                                f"Model Parameter {parameter_name} should be round to {parameter_rule.precision} decimal places.")
+
+                # validate parameter value range
+                if parameter_rule.min is not None and parameter_value < parameter_rule.min:
+                    raise ValueError(
+                        f"Model Parameter {parameter_name} should be greater than or equal to {parameter_rule.min}.")
+
+                if parameter_rule.max is not None and parameter_value > parameter_rule.max:
+                    raise ValueError(
+                        f"Model Parameter {parameter_name} should be less than or equal to {parameter_rule.max}.")
+            elif parameter_rule.type == ParameterType.BOOLEAN:
+                if not isinstance(parameter_value, bool):
+                    raise ValueError(f"Model Parameter {parameter_name} should be bool.")
+            elif parameter_rule.type == ParameterType.STRING:
+                if not isinstance(parameter_value, str):
+                    raise ValueError(f"Model Parameter {parameter_name} should be string.")
+
+                # validate options
+                if parameter_rule.options and parameter_value not in parameter_rule.options:
+                    raise ValueError(f"Model Parameter {parameter_name} should be one of {parameter_rule.options}.")
+            else:
+                raise ValueError(f"Model Parameter {parameter_name} type {parameter_rule.type} is not supported.")
+
+            filtered_model_parameters[parameter_name] = parameter_value
+
+        return filtered_model_parameters

api/core/model_runtime/model_providers/__base/model_provider.py

@@ -0,0 +1,125 @@
+import importlib
+import os
+from abc import ABC, abstractmethod
+from typing import Optional, Dict
+
+import yaml
+
+from core.model_runtime.entities.model_entities import ModelType, AIModelEntity
+from core.model_runtime.entities.provider_entities import ProviderEntity
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+
+class ModelProvider(ABC):
+    provider_schema: ProviderEntity = None
+    model_instance_map: Dict[str, AIModel] = {}
+
+    @abstractmethod
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+        You can choose any validate_credentials method of model type or implement validate method by yourself,
+        such as: get model list api
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        raise NotImplementedError
+
+    def get_provider_schema(self) -> ProviderEntity:
+        """
+        Get provider schema
+
+        :return: provider schema
+        """
+        if self.provider_schema:
+            return self.provider_schema
+
+        # get dirname of the current path
+        provider_name = self.__class__.__module__.split('.')[-1]
+
+        # get the path of the model_provider classes
+        base_path = os.path.abspath(__file__)
+        current_path = os.path.join(os.path.dirname(os.path.dirname(base_path)), provider_name)
+
+        # read provider schema from yaml file
+        yaml_path = os.path.join(current_path, f'{provider_name}.yaml')
+        yaml_data = {}
+        if os.path.exists(yaml_path):
+            with open(yaml_path, 'r') as f:
+                yaml_data = yaml.safe_load(f)
+
+        try:
+            # yaml_data to entity
+            provider_schema = ProviderEntity(**yaml_data)
+        except Exception as e:
+            raise Exception(f'Invalid provider schema for {provider_name}: {str(e)}')
+
+        # cache schema
+        self.provider_schema = provider_schema
+
+        return provider_schema
+
+    def models(self, model_type: ModelType) -> list[AIModelEntity]:
+        """
+        Get all models for given model type
+
+        :param model_type: model type defined in `ModelType`
+        :return: list of models
+        """
+        provider_schema = self.get_provider_schema()
+        if model_type not in provider_schema.supported_model_types:
+            return []
+
+        # get model instance of the model type
+        model_instance = self.get_model_instance(model_type)
+
+        # get predefined models (predefined_models)
+        models = model_instance.predefined_models()
+
+        # return models
+        return models
+
+    def get_model_instance(self, model_type: ModelType) -> AIModel:
+        """
+        Get model instance
+
+        :param model_type: model type defined in `ModelType`
+        :return:
+        """
+        # get dirname of the current path
+        provider_name = self.__class__.__module__.split('.')[-1]
+
+        if f"{provider_name}.{model_type.value}" in self.model_instance_map:
+            return self.model_instance_map[f"{provider_name}.{model_type.value}"]
+
+        # get the path of the model type classes
+        base_path = os.path.abspath(__file__)
+        model_type_name = model_type.value.replace('-', '_')
+        model_type_path = os.path.join(os.path.dirname(os.path.dirname(base_path)), provider_name, model_type_name)
+        model_type_py_path = os.path.join(model_type_path, f'{model_type_name}.py')
+
+        if not os.path.isdir(model_type_path) or not os.path.exists(model_type_py_path):
+            raise Exception(f'Invalid model type {model_type} for provider {provider_name}')
+
+        # Dynamic loading {model_type_name}.py file and find the subclass of AIModel
+        parent_module = '.'.join(self.__class__.__module__.split('.')[:-1])
+        spec = importlib.util.spec_from_file_location(f"{parent_module}.{model_type_name}.{model_type_name}", model_type_py_path)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        model_class = None
+        for name, obj in vars(mod).items():
+            if (isinstance(obj, type) and issubclass(obj, AIModel) and not obj.__abstractmethods__
+                    and obj != AIModel):
+                model_class = obj
+                break
+
+        if not model_class:
+            raise Exception(f'Missing AIModel Class for model type {model_type} in {model_type_py_path}')
+
+        model_instance_map = model_class()
+        self.model_instance_map[f"{provider_name}.{model_type.value}"] = model_instance_map
+
+        return model_instance_map

api/core/model_runtime/model_providers/__base/moderation_model.py

@@ -0,0 +1,48 @@
+import time
+from abc import abstractmethod
+from typing import Optional
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+
+class ModerationModel(AIModel):
+    """
+    Model class for moderation model.
+    """
+    model_type: ModelType = ModelType.MODERATION
+
+    def invoke(self, model: str, credentials: dict,
+               text: str, user: Optional[str] = None) \
+            -> bool:
+        """
+        Invoke moderation model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param text: text to moderate
+        :param user: unique user id
+        :return: false if text is safe, true otherwise
+        """
+        self.started_at = time.perf_counter()
+
+        try:
+            return self._invoke(model, credentials, text, user)
+        except Exception as e:
+            raise self._transform_invoke_error(e)
+
+    @abstractmethod
+    def _invoke(self, model: str, credentials: dict,
+                text: str, user: Optional[str] = None) \
+            -> bool:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param text: text to moderate
+        :param user: unique user id
+        :return: false if text is safe, true otherwise
+        """
+        raise NotImplementedError
+

api/core/model_runtime/model_providers/__base/rerank_model.py

@@ -0,0 +1,56 @@
+import time
+from abc import abstractmethod
+from typing import Optional
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.entities.rerank_entities import RerankResult
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+
+class RerankModel(AIModel):
+    """
+    Base Model class for rerank model.
+    """
+    model_type: ModelType = ModelType.RERANK
+
+    def invoke(self, model: str, credentials: dict,
+               query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None,
+               user: Optional[str] = None) \
+            -> RerankResult:
+        """
+        Invoke rerank model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param query: search query
+        :param docs: docs for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n
+        :param user: unique user id
+        :return: rerank result
+        """
+        self.started_at = time.perf_counter()
+
+        try:
+            return self._invoke(model, credentials, query, docs, score_threshold, top_n, user)
+        except Exception as e:
+            raise self._transform_invoke_error(e)
+
+    @abstractmethod
+    def _invoke(self, model: str, credentials: dict,
+                query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None,
+                user: Optional[str] = None) \
+            -> RerankResult:
+        """
+        Invoke rerank model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param query: search query
+        :param docs: docs for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n
+        :param user: unique user id
+        :return: rerank result
+        """
+        raise NotImplementedError

api/core/model_runtime/model_providers/__base/speech2text_model.py

@@ -0,0 +1,57 @@
+import os
+from abc import abstractmethod
+from typing import Optional, IO
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+
+class Speech2TextModel(AIModel):
+    """
+    Model class for speech2text model.
+    """
+    model_type: ModelType = ModelType.SPEECH2TEXT
+
+    def invoke(self, model: str, credentials: dict,
+               file: IO[bytes], user: Optional[str] = None) \
+            -> str:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param file: audio file
+        :param user: unique user id
+        :return: text for given audio file
+        """
+        try:
+            return self._invoke(model, credentials, file, user)
+        except Exception as e:
+            raise self._transform_invoke_error(e)
+
+    @abstractmethod
+    def _invoke(self, model: str, credentials: dict,
+                file: IO[bytes], user: Optional[str] = None) \
+            -> str:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param file: audio file
+        :param user: unique user id
+        :return: text for given audio file
+        """
+        raise NotImplementedError
+
+    def _get_demo_file_path(self) -> str:
+        """
+        Get demo file for given model
+
+        :return: demo file
+        """
+        # Get the directory of the current file
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+
+        # Construct the path to the audio file
+        return os.path.join(current_dir, 'audio.mp3')

api/core/model_runtime/model_providers/__base/text_embedding_model.py

@@ -0,0 +1,90 @@
+import time
+from abc import abstractmethod
+from typing import Optional
+
+from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
+from core.model_runtime.model_providers.__base.ai_model import AIModel
+
+
+class TextEmbeddingModel(AIModel):
+    """
+    Model class for text embedding model.
+    """
+    model_type: ModelType = ModelType.TEXT_EMBEDDING
+
+    def invoke(self, model: str, credentials: dict,
+               texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :param user: unique user id
+        :return: embeddings result
+        """
+        self.started_at = time.perf_counter()
+
+        try:
+            return self._invoke(model, credentials, texts, user)
+        except Exception as e:
+            raise self._transform_invoke_error(e)
+
+    @abstractmethod
+    def _invoke(self, model: str, credentials: dict,
+                texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :param user: unique user id
+        :return: embeddings result
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :return:
+        """
+        raise NotImplementedError
+
+    def _get_context_size(self, model: str, credentials: dict) -> int:
+        """
+        Get context size for given embedding model
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: context size
+        """
+        model_schema = self.get_model_schema(model, credentials)
+
+        if model_schema and ModelPropertyKey.CONTEXT_SIZE in model_schema.model_properties:
+            return model_schema.model_properties[ModelPropertyKey.CONTEXT_SIZE]
+
+        return 1000
+
+    def _get_max_chunks(self, model: str, credentials: dict) -> int:
+        """
+        Get max chunks for given embedding model
+
+        :param model: model name
+        :param credentials: model credentials
+        :return: max chunks
+        """
+        model_schema = self.get_model_schema(model, credentials)
+
+        if model_schema and ModelPropertyKey.MAX_CHUNKS in model_schema.model_properties:
+            return model_schema.model_properties[ModelPropertyKey.MAX_CHUNKS]
+
+        return 1

api/core/model_runtime/model_providers/__base/tokenizers/gpt2/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

api/core/model_runtime/model_providers/__base/tokenizers/gpt2/tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py

@@ -0,0 +1,32 @@
+from transformers import GPT2Tokenizer as TransformerGPT2Tokenizer
+from os.path import join, abspath, dirname
+from typing import Any
+from threading import Lock
+
+_tokenizer = None
+_lock = Lock()
+
+class GPT2Tokenizer:
+    @staticmethod
+    def _get_num_tokens_by_gpt2(text: str) -> int:
+        """
+            use gpt2 tokenizer to get num tokens
+        """
+        _tokenizer = GPT2Tokenizer.get_encoder()
+        tokens = _tokenizer.encode(text, verbose=False)
+        return len(tokens)
+    
+    @staticmethod
+    def get_num_tokens(text: str) -> int:
+        return GPT2Tokenizer._get_num_tokens_by_gpt2(text)
+    
+    @staticmethod
+    def get_encoder() -> Any:
+        global _tokenizer, _lock
+        with _lock:
+            if _tokenizer is None:
+                base_path = abspath(__file__)
+                gpt2_tokenizer_path = join(dirname(base_path), 'gpt2')
+                _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
+
+            return _tokenizer

api/core/model_runtime/model_providers/__init__.py

@@ -0,0 +1,3 @@
+from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
+
+model_provider_factory = ModelProviderFactory()

api/core/model_runtime/model_providers/_position.yaml

@@ -0,0 +1,19 @@
+openai: 0
+anthropic: 1
+azure_openai: 2
+google: 3
+replicate: 4
+huggingface_hub: 5
+cohere: 6
+zhipuai: 7
+baichuan: 8
+spark: 9
+minimax: 10
+tongyi: 11
+wenxin: 12
+jina: 13
+chatglm: 14
+xinference: 15
+openllm: 16
+localai: 17
+openai_api_compatible: 18

api/core/model_runtime/model_providers/anthropic/_assets/icon_l_en.svg

@@ -0,0 +1,78 @@
+<svg width="90" height="20" viewBox="0 0 90 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_8587_60274)">
+<mask id="mask0_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M89.375 4.99805H0V14.998H89.375V4.99805Z" fill="white"/>
+</mask>
+<g mask="url(#mask0_8587_60274)">
+<mask id="mask1_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99609H89.375V14.9961H0V4.99609Z" fill="white"/>
+</mask>
+<g mask="url(#mask1_8587_60274)">
+<mask id="mask2_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99414H89.375V14.9941H0V4.99414Z" fill="white"/>
+</mask>
+<g mask="url(#mask2_8587_60274)">
+<mask id="mask3_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask3_8587_60274)">
+<path d="M18.1273 11.9244L13.7773 5.15625H11.4297V14.825H13.4321V8.05688L17.7821 14.825H20.1297V5.15625H18.1273V11.9244Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask4_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask4_8587_60274)">
+<path d="M21.7969 7.02094H25.0423V14.825H27.1139V7.02094H30.3594V5.15625H21.7969V7.02094Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask5_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask5_8587_60274)">
+<path d="M38.6442 9.00994H34.0871V5.15625H32.0156V14.825H34.0871V10.8746H38.6442V14.825H40.7156V5.15625H38.6442V9.00994Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask6_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask6_8587_60274)">
+<path d="M45.3376 7.02094H47.893C48.9152 7.02094 49.4539 7.39387 49.4539 8.09831C49.4539 8.80275 48.9152 9.17569 47.893 9.17569H45.3376V7.02094ZM51.5259 8.09831C51.5259 6.27506 50.186 5.15625 47.9897 5.15625H43.2656V14.825H45.3376V11.0404H47.6443L49.7164 14.825H52.0094L49.715 10.7521C50.8666 10.3094 51.5259 9.37721 51.5259 8.09831Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask7_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask7_8587_60274)">
+<path d="M57.8732 13.0565C56.2438 13.0565 55.2496 11.8963 55.2496 10.004C55.2496 8.08416 56.2438 6.92394 57.8732 6.92394C59.4887 6.92394 60.4691 8.08416 60.4691 10.004C60.4691 11.8963 59.4887 13.0565 57.8732 13.0565ZM57.8732 4.99023C55.0839 4.99023 53.1094 7.06206 53.1094 10.004C53.1094 12.9184 55.0839 14.9902 57.8732 14.9902C60.6486 14.9902 62.6094 12.9184 62.6094 10.004C62.6094 7.06206 60.6486 4.99023 57.8732 4.99023Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask8_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask8_8587_60274)">
+<path d="M69.1794 9.45194H66.6233V7.02094H69.1794C70.2019 7.02094 70.7407 7.43532 70.7407 8.23644C70.7407 9.03756 70.2019 9.45194 69.1794 9.45194ZM69.2762 5.15625H64.5508V14.825H66.6233V11.3166H69.2762C71.473 11.3166 72.8133 10.1564 72.8133 8.23644C72.8133 6.3165 71.473 5.15625 69.2762 5.15625Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask9_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask9_8587_60274)">
+<path d="M86.8413 11.5786C86.4823 12.5179 85.7642 13.0565 84.7837 13.0565C83.1542 13.0565 82.16 11.8963 82.16 10.004C82.16 8.08416 83.1542 6.92394 84.7837 6.92394C85.7642 6.92394 86.4823 7.46261 86.8413 8.40183H89.0369C88.4984 6.33002 86.8827 4.99023 84.7837 4.99023C81.9942 4.99023 80.0195 7.06206 80.0195 10.004C80.0195 12.9184 81.9942 14.9902 84.7837 14.9902C86.8965 14.9902 88.5122 13.6366 89.0508 11.5786H86.8413Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask10_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask10_8587_60274)">
+<path d="M73.6484 5.15625L77.5033 14.825H79.6172L75.7624 5.15625H73.6484Z" fill="black" fill-opacity="0.92"/>
+</g>
+<mask id="mask11_8587_60274" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="4" width="90" height="11">
+<path d="M0 4.99219H89.375V14.9922H0V4.99219Z" fill="white"/>
+</mask>
+<g mask="url(#mask11_8587_60274)">
+<path d="M3.64038 10.9989L4.95938 7.60106L6.27838 10.9989H3.64038ZM3.85422 5.15625L0 14.825H2.15505L2.9433 12.7946H6.97558L7.76371 14.825H9.91875L6.06453 5.15625H3.85422Z" fill="black" fill-opacity="0.92"/>
+</g>
+</g>
+</g>
+</g>
+</g>
+<defs>
+<clipPath id="clip0_8587_60274">
+<rect width="89.375" height="10" fill="white" transform="translate(0 5)"/>
+</clipPath>
+</defs>
+</svg>

api/core/model_runtime/model_providers/anthropic/_assets/icon_s_en.svg

@@ -0,0 +1,4 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="24" height="24" rx="6" fill="#CA9F7B"/>
+<path d="M15.3843 6.43481H12.9687L17.3739 17.5652H19.7896L15.3843 6.43481ZM8.40522 6.43481L4 17.5652H6.4633L7.36417 15.2279H11.9729L12.8737 17.5652H15.337L10.9318 6.43481H8.40522ZM8.16104 13.1607L9.66852 9.24907L11.176 13.1607H8.16104Z" fill="#191918"/>
+</svg>

api/core/model_runtime/model_providers/anthropic/anthropic.py

@@ -0,0 +1,31 @@
+import logging
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class AnthropicProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.LLM)
+
+            # Use `claude-instant-1` model for validate,
+            model_instance.validate_credentials(
+                model='claude-instant-1',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/anthropic/anthropic.yaml

@@ -0,0 +1,39 @@
+provider: anthropic
+label:
+  en_US: Anthropic
+description:
+  en_US: Anthropic’s powerful models, such as Claude 2 and Claude Instant.
+  zh_Hans: Anthropic 的强大模型，例如 Claude 2 和 Claude Instant。
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#F0F0EB"
+help:
+  title:
+    en_US: Get your API Key from Anthropic
+    zh_Hans: 从 Anthropic 获取 API Key
+  url:
+    en_US: https://console.anthropic.com/account/keys
+supported_model_types:
+- llm
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: anthropic_api_key
+    label:
+      en_US: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API Key
+      en_US: Enter your API Key
+  - variable: anthropic_api_url
+    label:
+      en_US: API URL
+    type: text-input
+    required: false
+    placeholder:
+      zh_Hans: 在此输入您的 API URL
+      en_US: Enter your API URL

api/core/model_runtime/model_providers/anthropic/llm/claude-2.1.yaml

@@ -0,0 +1,34 @@
+model: claude-2.1
+label:
+  en_US: claude-2.1
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 200000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens_to_sample
+  use_template: max_tokens
+  required: true
+  default: 4096
+  min: 1
+  max: 4096
+pricing:
+  input: '8.00'
+  output: '24.00'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/anthropic/llm/claude-2.yaml

@@ -0,0 +1,34 @@
+model: claude-2
+label:
+  en_US: claude-2
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 100000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens_to_sample
+  use_template: max_tokens
+  required: true
+  default: 4096
+  min: 1
+  max: 4096
+pricing:
+  input: '8.00'
+  output: '24.00'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/anthropic/llm/claude-instant-1.yaml

@@ -0,0 +1,33 @@
+model: claude-instant-1
+label:
+  en_US: claude-instant-1
+model_type: llm
+features: []
+model_properties:
+  mode: chat
+  context_size: 100000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens_to_sample
+  use_template: max_tokens
+  required: true
+  default: 4096
+  min: 1
+  max: 4096
+pricing:
+  input: '1.63'
+  output: '5.51'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/anthropic/llm/llm.py

@@ -0,0 +1,298 @@
+from typing import Optional, Generator, Union, List
+
+import anthropic
+from anthropic import Anthropic, Stream
+from anthropic.types import completion_create_params, Completion
+from httpx import Timeout
+
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
+    SystemPromptMessage
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
+    LLMResultChunkDelta
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+
+
+class AnthropicLargeLanguageModel(LargeLanguageModel):
+
+    def _invoke(self, model: str, credentials: dict,
+                prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+                stream: bool = True, user: Optional[str] = None) \
+            -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        # invoke model
+        return self._generate(model, credentials, prompt_messages, model_parameters, stop, stream, user)
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param tools: tools for tool calling
+        :return:
+        """
+        prompt = self._convert_messages_to_prompt_anthropic(prompt_messages)
+
+        client = Anthropic(api_key="")
+        return client.count_tokens(prompt)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._generate(
+                model=model,
+                credentials=credentials,
+                prompt_messages=[
+                    UserPromptMessage(content="ping"),
+                ],
+                model_parameters={
+                    "temperature": 0,
+                    "max_tokens_to_sample": 20,
+                },
+                stream=False
+            )
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    def _generate(self, model: str, credentials: dict,
+                  prompt_messages: list[PromptMessage], model_parameters: dict,
+                  stop: Optional[List[str]] = None, stream: bool = True,
+                  user: Optional[str] = None) -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: credentials kwargs
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        # transform credentials to kwargs for model instance
+        credentials_kwargs = self._to_credential_kwargs(credentials)
+
+        client = Anthropic(**credentials_kwargs)
+
+        extra_model_kwargs = {}
+        if stop:
+            extra_model_kwargs['stop_sequences'] = stop
+
+        if user:
+            extra_model_kwargs['metadata'] = completion_create_params.Metadata(user_id=user)
+
+        response = client.completions.create(
+            model=model,
+            prompt=self._convert_messages_to_prompt_anthropic(prompt_messages),
+            stream=stream,
+            **model_parameters,
+            **extra_model_kwargs
+        )
+
+        if stream:
+            return self._handle_generate_stream_response(model, credentials, response, prompt_messages)
+
+        return self._handle_generate_response(model, credentials, response, prompt_messages)
+
+    def _handle_generate_response(self, model: str, credentials: dict, response: Completion,
+                                  prompt_messages: list[PromptMessage]) -> LLMResult:
+        """
+        Handle llm response
+
+        :param model: model name
+        :param credentials: credentials
+        :param response: response
+        :param prompt_messages: prompt messages
+        :return: llm response
+        """
+        # transform assistant message to prompt message
+        assistant_prompt_message = AssistantPromptMessage(
+            content=response.completion
+        )
+
+        # calculate num tokens
+        prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+        completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+        # transform usage
+        usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+        # transform response
+        result = LLMResult(
+            model=response.model,
+            prompt_messages=prompt_messages,
+            message=assistant_prompt_message,
+            usage=usage,
+        )
+
+        return result
+
+    def _handle_generate_stream_response(self, model: str, credentials: dict, response: Stream[Completion],
+                                         prompt_messages: list[PromptMessage]) -> Generator:
+        """
+        Handle llm stream response
+
+        :param model: model name
+        :param credentials: credentials
+        :param response: response
+        :param prompt_messages: prompt messages
+        :return: llm response chunk generator result
+        """
+        index = -1
+        for chunk in response:
+            content = chunk.completion
+            if chunk.stop_reason is None and (content is None or content == ''):
+                continue
+
+            # transform assistant message to prompt message
+            assistant_prompt_message = AssistantPromptMessage(
+                content=content if content else '',
+            )
+
+            index += 1
+
+            if chunk.stop_reason is not None:
+                # calculate num tokens
+                prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+                completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+                # transform usage
+                usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=index,
+                        message=assistant_prompt_message,
+                        finish_reason=chunk.stop_reason,
+                        usage=usage
+                    )
+                )
+            else:
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=index,
+                        message=assistant_prompt_message
+                    )
+                )
+
+    def _to_credential_kwargs(self, credentials: dict) -> dict:
+        """
+        Transform credentials to kwargs for model instance
+
+        :param credentials:
+        :return:
+        """
+        credentials_kwargs = {
+            "api_key": credentials['anthropic_api_key'],
+            "timeout": Timeout(315.0, read=300.0, write=10.0, connect=5.0),
+            "max_retries": 1,
+        }
+
+        if 'anthropic_api_url' in credentials and credentials['anthropic_api_url']:
+            credentials['anthropic_api_url'] = credentials['anthropic_api_url'].rstrip('/')
+            credentials_kwargs['base_url'] = credentials['anthropic_api_url']
+
+        return credentials_kwargs
+
+    def _convert_one_message_to_text(self, message: PromptMessage) -> str:
+        """
+        Convert a single message to a string.
+
+        :param message: PromptMessage to convert.
+        :return: String representation of the message.
+        """
+        human_prompt = "\n\nHuman:"
+        ai_prompt = "\n\nAssistant:"
+        content = message.content
+
+        if isinstance(message, UserPromptMessage):
+            message_text = f"{human_prompt} {content}"
+        elif isinstance(message, AssistantPromptMessage):
+            message_text = f"{ai_prompt} {content}"
+        elif isinstance(message, SystemPromptMessage):
+            message_text = content
+        else:
+            raise ValueError(f"Got unknown type {message}")
+
+        return message_text
+
+    def _convert_messages_to_prompt_anthropic(self, messages: List[PromptMessage]) -> str:
+        """
+        Format a list of messages into a full prompt for the Anthropic model
+
+        :param messages: List of PromptMessage to combine.
+        :return: Combined string with necessary human_prompt and ai_prompt tags.
+        """
+        messages = messages.copy()  # don't mutate the original list
+        if not isinstance(messages[-1], AssistantPromptMessage):
+            messages.append(AssistantPromptMessage(content=""))
+
+        text = "".join(
+            self._convert_one_message_to_text(message)
+            for message in messages
+        )
+
+        # trim off the trailing ' ' that might come from the "Assistant: "
+        return text.rstrip()
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        return {
+            InvokeConnectionError: [
+                anthropic.APIConnectionError,
+                anthropic.APITimeoutError
+            ],
+            InvokeServerUnavailableError: [
+                anthropic.InternalServerError
+            ],
+            InvokeRateLimitError: [
+                anthropic.RateLimitError
+            ],
+            InvokeAuthorizationError: [
+                anthropic.AuthenticationError,
+                anthropic.PermissionDeniedError
+            ],
+            InvokeBadRequestError: [
+                anthropic.BadRequestError,
+                anthropic.NotFoundError,
+                anthropic.UnprocessableEntityError,
+                anthropic.APIError
+            ]
+        }

api/core/model_runtime/model_providers/azure_openai/_assets/icon_s_en.svg

@@ -0,0 +1,8 @@
+<svg width="21" height="22" viewBox="0 0 21 22" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Microsfot">
+<rect id="Rectangle 1010" y="0.5" width="10" height="10" fill="#EF4F21"/>
+<rect id="Rectangle 1012" y="11.5" width="10" height="10" fill="#03A4EE"/>
+<rect id="Rectangle 1011" x="11" y="0.5" width="10" height="10" fill="#7EB903"/>
+<rect id="Rectangle 1013" x="11" y="11.5" width="10" height="10" fill="#FBB604"/>
+</g>
+</svg>

api/core/model_runtime/model_providers/azure_openai/_common.py

@@ -0,0 +1,46 @@
+import openai
+from httpx import Timeout
+
+from core.model_runtime.model_providers.azure_openai._constant import AZURE_OPENAI_API_VERSION
+
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+
+
+class _CommonAzureOpenAI:
+    @staticmethod
+    def _to_credential_kwargs(credentials: dict) -> dict:
+        credentials_kwargs = {
+            "api_key": credentials['openai_api_key'],
+            "azure_endpoint": credentials['openai_api_base'],
+            "api_version": AZURE_OPENAI_API_VERSION,
+            "timeout": Timeout(315.0, read=300.0, write=10.0, connect=5.0),
+            "max_retries": 1,
+        }
+
+        return credentials_kwargs
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        return {
+            InvokeConnectionError: [
+                openai.APIConnectionError,
+                openai.APITimeoutError
+            ],
+            InvokeServerUnavailableError: [
+                openai.InternalServerError
+            ],
+            InvokeRateLimitError: [
+                openai.RateLimitError
+            ],
+            InvokeAuthorizationError: [
+                openai.AuthenticationError,
+                openai.PermissionDeniedError
+            ],
+            InvokeBadRequestError: [
+                openai.BadRequestError,
+                openai.NotFoundError,
+                openai.UnprocessableEntityError,
+                openai.APIError
+            ]
+        }

api/core/model_runtime/model_providers/azure_openai/_constant.py

@@ -0,0 +1,475 @@
+from pydantic import BaseModel
+
+from core.model_runtime.entities.llm_entities import LLMMode
+from core.model_runtime.entities.model_entities import ModelFeature, ModelType, FetchFrom, ParameterRule, \
+    DefaultParameterName, PriceConfig
+from core.model_runtime.entities.model_entities import AIModelEntity, I18nObject
+from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
+
+AZURE_OPENAI_API_VERSION = '2023-12-01-preview'
+
+
+def _get_max_tokens(default: int, min_val: int, max_val: int) -> ParameterRule:
+    rule = ParameterRule(
+        name='max_tokens',
+        **PARAMETER_RULE_TEMPLATE[DefaultParameterName.MAX_TOKENS],
+    )
+    rule.default = default
+    rule.min = min_val
+    rule.max = max_val
+    return rule
+
+
+class AzureBaseModel(BaseModel):
+    base_model_name: str
+    entity: AIModelEntity
+
+
+LLM_BASE_MODELS = [
+    AzureBaseModel(
+        base_model_name='gpt-35-turbo',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.AGENT_THOUGHT,
+                ModelFeature.MULTI_TOOL_CALL,
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 4096,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=4096)
+            ],
+            pricing=PriceConfig(
+                input=0.001,
+                output=0.002,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-35-turbo-16k',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.AGENT_THOUGHT,
+                ModelFeature.MULTI_TOOL_CALL,
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 16385,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=16385)
+            ],
+            pricing=PriceConfig(
+                input=0.003,
+                output=0.004,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-4',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.AGENT_THOUGHT,
+                ModelFeature.MULTI_TOOL_CALL,
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 8192,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=8192),
+                ParameterRule(
+                    name='seed',
+                    label=I18nObject(
+                        zh_Hans='种子',
+                        en_US='Seed'
+                    ),
+                    type='int',
+                    help=I18nObject(
+                        zh_Hans='如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。',
+                        en_US='If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.'
+                    ),
+                    required=False,
+                    precision=2,
+                    min=0,
+                    max=1,
+                ),
+                ParameterRule(
+                    name='response_format',
+                    label=I18nObject(
+                        zh_Hans='回复格式',
+                        en_US='response_format'
+                    ),
+                    type='string',
+                    help=I18nObject(
+                        zh_Hans='指定模型必须输出的格式',
+                        en_US='specifying the format that the model must output'
+                    ),
+                    required=False,
+                    options=['text', 'json_object']
+                ),
+            ],
+            pricing=PriceConfig(
+                input=0.03,
+                output=0.06,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-4-32k',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.AGENT_THOUGHT,
+                ModelFeature.MULTI_TOOL_CALL,
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 32768,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=32768),
+                ParameterRule(
+                    name='seed',
+                    label=I18nObject(
+                        zh_Hans='种子',
+                        en_US='Seed'
+                    ),
+                    type='int',
+                    help=I18nObject(
+                        zh_Hans='如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。',
+                        en_US='If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.'
+                    ),
+                    required=False,
+                    precision=2,
+                    min=0,
+                    max=1,
+                ),
+                ParameterRule(
+                    name='response_format',
+                    label=I18nObject(
+                        zh_Hans='回复格式',
+                        en_US='response_format'
+                    ),
+                    type='string',
+                    help=I18nObject(
+                        zh_Hans='指定模型必须输出的格式',
+                        en_US='specifying the format that the model must output'
+                    ),
+                    required=False,
+                    options=['text', 'json_object']
+                ),
+            ],
+            pricing=PriceConfig(
+                input=0.06,
+                output=0.12,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-4-1106-preview',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.AGENT_THOUGHT,
+                ModelFeature.MULTI_TOOL_CALL,
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 128000,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=128000),
+                ParameterRule(
+                    name='seed',
+                    label=I18nObject(
+                        zh_Hans='种子',
+                        en_US='Seed'
+                    ),
+                    type='int',
+                    help=I18nObject(
+                        zh_Hans='如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。',
+                        en_US='If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.'
+                    ),
+                    required=False,
+                    precision=2,
+                    min=0,
+                    max=1,
+                ),
+                ParameterRule(
+                    name='response_format',
+                    label=I18nObject(
+                        zh_Hans='回复格式',
+                        en_US='response_format'
+                    ),
+                    type='string',
+                    help=I18nObject(
+                        zh_Hans='指定模型必须输出的格式',
+                        en_US='specifying the format that the model must output'
+                    ),
+                    required=False,
+                    options=['text', 'json_object']
+                ),
+            ],
+            pricing=PriceConfig(
+                input=0.01,
+                output=0.03,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-4-vision-preview',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            features=[
+                ModelFeature.VISION
+            ],
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.CHAT.value,
+                'context_size': 128000,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=128000),
+                ParameterRule(
+                    name='seed',
+                    label=I18nObject(
+                        zh_Hans='种子',
+                        en_US='Seed'
+                    ),
+                    type='int',
+                    help=I18nObject(
+                        zh_Hans='如果指定，模型将尽最大努力进行确定性采样，使得重复的具有相同种子和参数的请求应该返回相同的结果。不能保证确定性，您应该参考 system_fingerprint 响应参数来监视变化。',
+                        en_US='If specified, model will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed, and you should refer to the system_fingerprint response parameter to monitor changes in the backend.'
+                    ),
+                    required=False,
+                    precision=2,
+                    min=0,
+                    max=1,
+                ),
+                ParameterRule(
+                    name='response_format',
+                    label=I18nObject(
+                        zh_Hans='回复格式',
+                        en_US='response_format'
+                    ),
+                    type='string',
+                    help=I18nObject(
+                        zh_Hans='指定模型必须输出的格式',
+                        en_US='specifying the format that the model must output'
+                    ),
+                    required=False,
+                    options=['text', 'json_object']
+                ),
+            ],
+            pricing=PriceConfig(
+                input=0.01,
+                output=0.03,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    ),
+    AzureBaseModel(
+        base_model_name='gpt-35-turbo-instruct',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label',
+            ),
+            model_type=ModelType.LLM,
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_properties={
+                'mode': LLMMode.COMPLETION.value,
+                'context_size': 4096,
+            },
+            parameter_rules=[
+                ParameterRule(
+                    name='temperature',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TEMPERATURE],
+                ),
+                ParameterRule(
+                    name='top_p',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.TOP_P],
+                ),
+                ParameterRule(
+                    name='presence_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.PRESENCE_PENALTY],
+                ),
+                ParameterRule(
+                    name='frequency_penalty',
+                    **PARAMETER_RULE_TEMPLATE[DefaultParameterName.FREQUENCY_PENALTY],
+                ),
+                _get_max_tokens(default=512, min_val=1, max_val=4096),
+            ],
+            pricing=PriceConfig(
+                input=0.0015,
+                output=0.002,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    )
+]
+
+EMBEDDING_BASE_MODELS = [
+    AzureBaseModel(
+        base_model_name='text-embedding-ada-002',
+        entity=AIModelEntity(
+            model='fake-deployment-name',
+            label=I18nObject(
+                en_US='fake-deployment-name-label'
+            ),
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_type=ModelType.TEXT_EMBEDDING,
+            model_properties={
+                'context_size': 8097,
+                'max_chunks': 32,
+            },
+            pricing=PriceConfig(
+                input=0.0001,
+                unit=0.001,
+                currency='USD',
+            )
+        )
+    )
+]

api/core/model_runtime/model_providers/azure_openai/azure_openai.py

@@ -0,0 +1,11 @@
+import logging
+
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class AzureOpenAIProvider(ModelProvider):
+
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        pass

api/core/model_runtime/model_providers/azure_openai/azure_openai.yaml

@@ -0,0 +1,104 @@
+provider: azure_openai
+label:
+  en_US: Azure OpenAI Service Model
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.png
+background: "#E3F0FF"
+help:
+  title:
+    en_US: Get your API key from Azure
+    zh_Hans: 从 Azure 获取 API Key
+  url:
+    en_US: https://azure.microsoft.com/en-us/products/ai-services/openai-service
+supported_model_types:
+- llm
+- text-embedding
+configurate_methods:
+- customizable-model
+model_credential_schema:
+  model:
+    label:
+      en_US: Deployment Name
+      zh_Hans: 部署名称
+    placeholder:
+      en_US: Enter your Deployment Name here, matching the Azure deployment name.
+      zh_Hans: 在此输入您的部署名称，与 Azure 部署名称匹配。
+  credential_form_schemas:
+  - variable: openai_api_base
+    label:
+      en_US: API Endpoint URL
+      zh_Hans: API 域名
+    type: text-input
+    required: true
+    placeholder:
+      zh_Hans: '在此输入您的 API 域名，如：https://example.com/xxx'
+      en_US: 'Enter your API Endpoint, eg: https://example.com/xxx'
+  - variable: openai_api_key
+    label:
+      en_US: API Key
+      zh_Hans: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API Key
+      en_US: Enter your API key here
+  - variable: base_model_name
+    label:
+      en_US: Base Model
+      zh_Hans: 基础模型
+    type: select
+    required: true
+    options:
+    - label:
+        en_US: gpt-35-turbo
+      value: gpt-35-turbo
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-35-turbo-16k
+      value: gpt-35-turbo-16k
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-4
+      value: gpt-4
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-4-32k
+      value: gpt-4-32k
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-4-1106-preview
+      value: gpt-4-1106-preview
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-4-vision-preview
+      value: gpt-4-vision-preview
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: gpt-35-turbo-instruct
+      value: gpt-35-turbo-instruct
+      show_on:
+      - variable: __model_type
+        value: llm
+    - label:
+        en_US: text-embedding-ada-002
+      value: text-embedding-ada-002
+      show_on:
+        - variable: __model_type
+          value: text-embedding
+    placeholder:
+      zh_Hans: 在此输入您的模型版本
+      en_US: Enter your model version

api/core/model_runtime/model_providers/azure_openai/llm/llm.py

@@ -0,0 +1,627 @@
+import logging
+from typing import Optional, Generator, Union, List, cast
+
+import tiktoken
+from openai import AzureOpenAI, Stream
+from openai.types import Completion
+from openai.types.chat import ChatCompletionChunk, ChatCompletion, ChatCompletionMessageToolCall
+from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall, ChoiceDeltaFunctionCall
+from openai.types.chat.chat_completion_message import FunctionCall
+
+from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, \
+    LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \
+    UserPromptMessage, PromptMessageContentType, ImagePromptMessageContent, \
+    TextPromptMessageContent, SystemPromptMessage, ToolPromptMessage
+from core.model_runtime.entities.model_entities import AIModelEntity, ModelPropertyKey
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.model_providers.azure_openai._common import _CommonAzureOpenAI
+from core.model_runtime.model_providers.azure_openai._constant import LLM_BASE_MODELS, AzureBaseModel
+
+logger = logging.getLogger(__name__)
+
+
+class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel):
+
+    def _invoke(self, model: str, credentials: dict,
+                prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+                stream: bool = True, user: Optional[str] = None) \
+            -> Union[LLMResult, Generator]:
+
+        ai_model_entity = self._get_ai_model_entity(credentials['base_model_name'], model)
+
+        if ai_model_entity.entity.model_properties.get(ModelPropertyKey.MODE) == LLMMode.CHAT.value:
+            # chat model
+            return self._chat_generate(
+                model=model,
+                credentials=credentials,
+                prompt_messages=prompt_messages,
+                model_parameters=model_parameters,
+                tools=tools,
+                stop=stop,
+                stream=stream,
+                user=user
+            )
+        else:
+            # text completion model
+            return self._generate(
+                model=model,
+                credentials=credentials,
+                prompt_messages=prompt_messages,
+                model_parameters=model_parameters,
+                stop=stop,
+                stream=stream,
+                user=user
+            )
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: Optional[list[PromptMessageTool]] = None) -> int:
+
+        model_mode = self._get_ai_model_entity(credentials['base_model_name'], model).entity.model_properties.get(
+            ModelPropertyKey.MODE)
+
+        if model_mode == LLMMode.CHAT.value:
+            # chat model
+            return self._num_tokens_from_messages(credentials, prompt_messages, tools)
+        else:
+            # text completion model, do not support tool calling
+            return self._num_tokens_from_string(credentials, prompt_messages[0].content)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        if 'openai_api_base' not in credentials:
+            raise CredentialsValidateFailedError('Azure OpenAI API Base Endpoint is required')
+
+        if 'openai_api_key' not in credentials:
+            raise CredentialsValidateFailedError('Azure OpenAI API key is required')
+
+        if 'base_model_name' not in credentials:
+            raise CredentialsValidateFailedError('Base Model Name is required')
+
+        ai_model_entity = self._get_ai_model_entity(credentials['base_model_name'], model)
+
+        if not ai_model_entity:
+            raise CredentialsValidateFailedError(f'Base Model Name {credentials["base_model_name"]} is invalid')
+
+        try:
+            client = AzureOpenAI(**self._to_credential_kwargs(credentials))
+
+            if ai_model_entity.entity.model_properties.get(ModelPropertyKey.MODE) == LLMMode.CHAT.value:
+                # chat model
+                client.chat.completions.create(
+                    messages=[{"role": "user", "content": 'ping'}],
+                    model=model,
+                    temperature=0,
+                    max_tokens=20,
+                    stream=False,
+                )
+            else:
+                # text completion model
+                client.completions.create(
+                    prompt='ping',
+                    model=model,
+                    temperature=0,
+                    max_tokens=20,
+                    stream=False,
+                )
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        ai_model_entity = self._get_ai_model_entity(credentials['base_model_name'], model)
+        return ai_model_entity.entity
+
+    def _generate(self, model: str, credentials: dict,
+                  prompt_messages: list[PromptMessage], model_parameters: dict, stop: Optional[List[str]] = None,
+                  stream: bool = True, user: Optional[str] = None) -> Union[LLMResult, Generator]:
+
+        client = AzureOpenAI(**self._to_credential_kwargs(credentials))
+
+        extra_model_kwargs = {}
+
+        if stop:
+            extra_model_kwargs['stop'] = stop
+
+        if user:
+            extra_model_kwargs['user'] = user
+
+        # text completion model
+        response = client.completions.create(
+            prompt=prompt_messages[0].content,
+            model=model,
+            stream=stream,
+            **model_parameters,
+            **extra_model_kwargs
+        )
+
+        if stream:
+            return self._handle_generate_stream_response(model, credentials, response, prompt_messages)
+
+        return self._handle_generate_response(model, credentials, response, prompt_messages)
+
+    def _handle_generate_response(self, model: str, credentials: dict, response: Completion,
+                                  prompt_messages: list[PromptMessage]) -> LLMResult:
+        assistant_text = response.choices[0].text
+
+        # transform assistant message to prompt message
+        assistant_prompt_message = AssistantPromptMessage(
+            content=assistant_text
+        )
+
+        # calculate num tokens
+        if response.usage:
+            # transform usage
+            prompt_tokens = response.usage.prompt_tokens
+            completion_tokens = response.usage.completion_tokens
+        else:
+            # calculate num tokens
+            prompt_tokens = self._num_tokens_from_string(credentials, prompt_messages[0].content)
+            completion_tokens = self._num_tokens_from_string(credentials, assistant_text)
+
+        # transform usage
+        usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+        # transform response
+        result = LLMResult(
+            model=response.model,
+            prompt_messages=prompt_messages,
+            message=assistant_prompt_message,
+            usage=usage,
+            system_fingerprint=response.system_fingerprint,
+        )
+
+        return result
+
+    def _handle_generate_stream_response(self, model: str, credentials: dict, response: Stream[Completion],
+                                         prompt_messages: list[PromptMessage]) -> Generator:
+        full_text = ''
+        for chunk in response:
+            if len(chunk.choices) == 0:
+                continue
+
+            delta = chunk.choices[0]
+
+            if delta.finish_reason is None and (delta.text is None or delta.text == ''):
+                continue
+
+            # transform assistant message to prompt message
+            text = delta.text if delta.text else ''
+            assistant_prompt_message = AssistantPromptMessage(
+                content=text
+            )
+
+            full_text += text
+
+            if delta.finish_reason is not None:
+                # calculate num tokens
+                if chunk.usage:
+                    # transform usage
+                    prompt_tokens = chunk.usage.prompt_tokens
+                    completion_tokens = chunk.usage.completion_tokens
+                else:
+                    # calculate num tokens
+                    prompt_tokens = self._num_tokens_from_string(credentials, prompt_messages[0].content)
+                    completion_tokens = self._num_tokens_from_string(credentials, full_text)
+
+                # transform usage
+                usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                        finish_reason=delta.finish_reason,
+                        usage=usage
+                    )
+                )
+            else:
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                    )
+                )
+
+    def _chat_generate(self, model: str, credentials: dict,
+                       prompt_messages: list[PromptMessage], model_parameters: dict,
+                       tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+                       stream: bool = True, user: Optional[str] = None) -> Union[LLMResult, Generator]:
+
+        client = AzureOpenAI(**self._to_credential_kwargs(credentials))
+
+        response_format = model_parameters.get("response_format")
+        if response_format:
+            if response_format == "json_object":
+                response_format = {"type": "json_object"}
+            else:
+                response_format = {"type": "text"}
+
+            model_parameters["response_format"] = response_format
+
+        extra_model_kwargs = {}
+
+        if tools:
+            # extra_model_kwargs['tools'] = [helper.dump_model(PromptMessageFunction(function=tool)) for tool in tools]
+            extra_model_kwargs['functions'] = [{
+                "name": tool.name,
+                "description": tool.description,
+                "parameters": tool.parameters
+            } for tool in tools]
+
+        if stop:
+            extra_model_kwargs['stop'] = stop
+
+        if user:
+            extra_model_kwargs['user'] = user
+
+        # chat model
+        response = client.chat.completions.create(
+            messages=[self._convert_prompt_message_to_dict(m) for m in prompt_messages],
+            model=model,
+            stream=stream,
+            **model_parameters,
+            **extra_model_kwargs,
+        )
+
+        if stream:
+            return self._handle_chat_generate_stream_response(model, credentials, response, prompt_messages, tools)
+
+        return self._handle_chat_generate_response(model, credentials, response, prompt_messages, tools)
+
+    def _handle_chat_generate_response(self, model: str, credentials: dict, response: ChatCompletion,
+                                       prompt_messages: list[PromptMessage],
+                                       tools: Optional[list[PromptMessageTool]] = None) -> LLMResult:
+
+        assistant_message = response.choices[0].message
+        # assistant_message_tool_calls = assistant_message.tool_calls
+        assistant_message_function_call = assistant_message.function_call
+
+        # extract tool calls from response
+        # tool_calls = self._extract_response_tool_calls(assistant_message_tool_calls)
+        function_call = self._extract_response_function_call(assistant_message_function_call)
+        tool_calls = [function_call] if function_call else []
+
+        # transform assistant message to prompt message
+        assistant_prompt_message = AssistantPromptMessage(
+            content=assistant_message.content,
+            tool_calls=tool_calls
+        )
+
+        # calculate num tokens
+        if response.usage:
+            # transform usage
+            prompt_tokens = response.usage.prompt_tokens
+            completion_tokens = response.usage.completion_tokens
+        else:
+            # calculate num tokens
+            prompt_tokens = self._num_tokens_from_messages(credentials, prompt_messages, tools)
+            completion_tokens = self._num_tokens_from_messages(credentials, [assistant_prompt_message])
+
+        # transform usage
+        usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+        # transform response
+        response = LLMResult(
+            model=response.model,
+            prompt_messages=prompt_messages,
+            message=assistant_prompt_message,
+            usage=usage,
+            system_fingerprint=response.system_fingerprint,
+        )
+
+        return response
+
+    def _handle_chat_generate_stream_response(self, model: str, credentials: dict,
+                                              response: Stream[ChatCompletionChunk],
+                                              prompt_messages: list[PromptMessage],
+                                              tools: Optional[list[PromptMessageTool]] = None) -> Generator:
+
+        full_assistant_content = ''
+        for chunk in response:
+            if len(chunk.choices) == 0:
+                continue
+
+            delta = chunk.choices[0]
+
+            if delta.finish_reason is None and (delta.delta.content is None or delta.delta.content == ''):
+                continue
+
+            # assistant_message_tool_calls = delta.delta.tool_calls
+            assistant_message_function_call = delta.delta.function_call
+
+            # extract tool calls from response
+            # tool_calls = self._extract_response_tool_calls(assistant_message_tool_calls)
+            function_call = self._extract_response_function_call(assistant_message_function_call)
+            tool_calls = [function_call] if function_call else []
+
+            # transform assistant message to prompt message
+            assistant_prompt_message = AssistantPromptMessage(
+                content=delta.delta.content if delta.delta.content else '',
+                tool_calls=tool_calls
+            )
+
+            full_assistant_content += delta.delta.content if delta.delta.content else ''
+
+            if delta.finish_reason is not None:
+                # calculate num tokens
+                prompt_tokens = self._num_tokens_from_messages(credentials, prompt_messages, tools)
+
+                full_assistant_prompt_message = AssistantPromptMessage(
+                    content=full_assistant_content,
+                    tool_calls=tool_calls
+                )
+                completion_tokens = self._num_tokens_from_messages(credentials, [full_assistant_prompt_message])
+
+                # transform usage
+                usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                        finish_reason=delta.finish_reason,
+                        usage=usage
+                    )
+                )
+            else:
+                yield LLMResultChunk(
+                    model=chunk.model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                    )
+                )
+
+    @staticmethod
+    def _extract_response_tool_calls(response_tool_calls: list[ChatCompletionMessageToolCall | ChoiceDeltaToolCall]) \
+            -> list[AssistantPromptMessage.ToolCall]:
+
+        tool_calls = []
+        if response_tool_calls:
+            for response_tool_call in response_tool_calls:
+                function = AssistantPromptMessage.ToolCall.ToolCallFunction(
+                    name=response_tool_call.function.name,
+                    arguments=response_tool_call.function.arguments
+                )
+
+                tool_call = AssistantPromptMessage.ToolCall(
+                    id=response_tool_call.id,
+                    type=response_tool_call.type,
+                    function=function
+                )
+                tool_calls.append(tool_call)
+
+        return tool_calls
+
+    @staticmethod
+    def _extract_response_function_call(response_function_call: FunctionCall | ChoiceDeltaFunctionCall) \
+            -> AssistantPromptMessage.ToolCall:
+
+        tool_call = None
+        if response_function_call:
+            function = AssistantPromptMessage.ToolCall.ToolCallFunction(
+                name=response_function_call.name,
+                arguments=response_function_call.arguments
+            )
+
+            tool_call = AssistantPromptMessage.ToolCall(
+                id=response_function_call.name,
+                type="function",
+                function=function
+            )
+
+        return tool_call
+
+    @staticmethod
+    def _convert_prompt_message_to_dict(message: PromptMessage) -> dict:
+
+        if isinstance(message, UserPromptMessage):
+            message = cast(UserPromptMessage, message)
+            if isinstance(message.content, str):
+                message_dict = {"role": "user", "content": message.content}
+            else:
+                sub_messages = []
+                for message_content in message.content:
+                    if message_content.type == PromptMessageContentType.TEXT:
+                        message_content = cast(TextPromptMessageContent, message_content)
+                        sub_message_dict = {
+                            "type": "text",
+                            "text": message_content.data
+                        }
+                        sub_messages.append(sub_message_dict)
+                    elif message_content.type == PromptMessageContentType.IMAGE:
+                        message_content = cast(ImagePromptMessageContent, message_content)
+                        sub_message_dict = {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": message_content.data,
+                                "detail": message_content.detail.value
+                            }
+                        }
+                        sub_messages.append(sub_message_dict)
+
+                message_dict = {"role": "user", "content": sub_messages}
+        elif isinstance(message, AssistantPromptMessage):
+            message = cast(AssistantPromptMessage, message)
+            message_dict = {"role": "assistant", "content": message.content}
+            if message.tool_calls:
+                # message_dict["tool_calls"] = [helper.dump_model(tool_call) for tool_call in
+                #                               message.tool_calls]
+                function_call = message.tool_calls[0]
+                message_dict["function_call"] = {
+                    "name": function_call.function.name,
+                    "arguments": function_call.function.arguments,
+                }
+        elif isinstance(message, SystemPromptMessage):
+            message = cast(SystemPromptMessage, message)
+            message_dict = {"role": "system", "content": message.content}
+        elif isinstance(message, ToolPromptMessage):
+            message = cast(ToolPromptMessage, message)
+            # message_dict = {
+            #     "role": "tool",
+            #     "content": message.content,
+            #     "tool_call_id": message.tool_call_id
+            # }
+            message_dict = {
+                "role": "function",
+                "content": message.content,
+                "name": message.tool_call_id
+            }
+        else:
+            raise ValueError(f"Got unknown type {message}")
+
+        if message.name is not None:
+            message_dict["name"] = message.name
+
+        return message_dict
+
+    def _num_tokens_from_string(self, credentials: dict, text: str,
+                                tools: Optional[list[PromptMessageTool]] = None) -> int:
+        try:
+            encoding = tiktoken.encoding_for_model(credentials['base_model_name'])
+        except KeyError:
+            encoding = tiktoken.get_encoding("cl100k_base")
+
+        num_tokens = len(encoding.encode(text))
+
+        if tools:
+            num_tokens += self._num_tokens_for_tools(encoding, tools)
+
+        return num_tokens
+
+    def _num_tokens_from_messages(self, credentials: dict, messages: List[PromptMessage],
+                                  tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """Calculate num tokens for gpt-3.5-turbo and gpt-4 with tiktoken package.
+
+        Official documentation: https://github.com/openai/openai-cookbook/blob/
+        main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb"""
+        model = credentials['base_model_name']
+        try:
+            encoding = tiktoken.encoding_for_model(model)
+        except KeyError:
+            logger.warning("Warning: model not found. Using cl100k_base encoding.")
+            model = "cl100k_base"
+            encoding = tiktoken.get_encoding(model)
+
+        if model.startswith("gpt-35-turbo-0301"):
+            # every message follows <im_start>{role/name}\n{content}<im_end>\n
+            tokens_per_message = 4
+            # if there's a name, the role is omitted
+            tokens_per_name = -1
+        elif model.startswith("gpt-35-turbo") or model.startswith("gpt-4"):
+            tokens_per_message = 3
+            tokens_per_name = 1
+        else:
+            raise NotImplementedError(
+                f"get_num_tokens_from_messages() is not presently implemented "
+                f"for model {model}."
+                "See https://github.com/openai/openai-python/blob/main/chatml.md for "
+                "information on how messages are converted to tokens."
+            )
+        num_tokens = 0
+        messages_dict = [self._convert_prompt_message_to_dict(m) for m in messages]
+        for message in messages_dict:
+            num_tokens += tokens_per_message
+            for key, value in message.items():
+                # Cast str(value) in case the message value is not a string
+                # This occurs with function messages
+                # TODO: The current token calculation method for the image type is not implemented,
+                #  which need to download the image and then get the resolution for calculation,
+                #  and will increase the request delay
+                if isinstance(value, list):
+                    text = ''
+                    for item in value:
+                        if isinstance(item, dict) and item['type'] == 'text':
+                            text += item['text']
+
+                    value = text
+
+                if key == "tool_calls":
+                    for tool_call in value:
+                        for t_key, t_value in tool_call.items():
+                            num_tokens += len(encoding.encode(t_key))
+                            if t_key == "function":
+                                for f_key, f_value in t_value.items():
+                                    num_tokens += len(encoding.encode(f_key))
+                                    num_tokens += len(encoding.encode(f_value))
+                            else:
+                                num_tokens += len(encoding.encode(t_key))
+                                num_tokens += len(encoding.encode(t_value))
+                else:
+                    num_tokens += len(encoding.encode(str(value)))
+
+                if key == "name":
+                    num_tokens += tokens_per_name
+
+        # every reply is primed with <im_start>assistant
+        num_tokens += 3
+
+        if tools:
+            num_tokens += self._num_tokens_for_tools(encoding, tools)
+
+        return num_tokens
+
+    @staticmethod
+    def _num_tokens_for_tools(encoding: tiktoken.Encoding, tools: list[PromptMessageTool]) -> int:
+
+        num_tokens = 0
+        for tool in tools:
+            num_tokens += len(encoding.encode('type'))
+            num_tokens += len(encoding.encode(tool.get("type")))
+            num_tokens += len(encoding.encode('function'))
+
+            # calculate num tokens for function object
+            num_tokens += len(encoding.encode('name'))
+            num_tokens += len(encoding.encode(tool.name))
+            num_tokens += len(encoding.encode('description'))
+            num_tokens += len(encoding.encode(tool.description))
+            parameters = tool.parameters
+            num_tokens += len(encoding.encode('parameters'))
+            if 'title' in parameters:
+                num_tokens += len(encoding.encode('title'))
+                num_tokens += len(encoding.encode(parameters.get("title")))
+            num_tokens += len(encoding.encode('type'))
+            num_tokens += len(encoding.encode(parameters.get("type")))
+            if 'properties' in parameters:
+                num_tokens += len(encoding.encode('properties'))
+                for key, value in parameters.get('properties').items():
+                    num_tokens += len(encoding.encode(key))
+                    for field_key, field_value in value.items():
+                        num_tokens += len(encoding.encode(field_key))
+                        if field_key == 'enum':
+                            for enum_field in field_value:
+                                num_tokens += 3
+                                num_tokens += len(encoding.encode(enum_field))
+                        else:
+                            num_tokens += len(encoding.encode(field_key))
+                            num_tokens += len(encoding.encode(str(field_value)))
+            if 'required' in parameters:
+                num_tokens += len(encoding.encode('required'))
+                for required_field in parameters['required']:
+                    num_tokens += 3
+                    num_tokens += len(encoding.encode(required_field))
+
+        return num_tokens
+
+    @staticmethod
+    def _get_ai_model_entity(base_model_name: str, model: str) -> AzureBaseModel:
+        for ai_model_entity in LLM_BASE_MODELS:
+            if ai_model_entity.base_model_name == base_model_name:
+                ai_model_entity.entity.model = model
+                ai_model_entity.entity.label.en_US = model
+                ai_model_entity.entity.label.zh_Hans = model
+                return ai_model_entity
+
+        return None

api/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py

@@ -0,0 +1,195 @@
+import base64
+import time
+from typing import Optional, Tuple
+
+import numpy as np
+import tiktoken
+from openai import AzureOpenAI
+
+from core.model_runtime.entities.model_entities import PriceType, AIModelEntity
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
+from core.model_runtime.model_providers.azure_openai._common import _CommonAzureOpenAI
+from core.model_runtime.model_providers.azure_openai._constant import EMBEDDING_BASE_MODELS, AzureBaseModel
+
+
+class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel):
+
+    def _invoke(self, model: str, credentials: dict,
+                texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        base_model_name = credentials['base_model_name']
+        credentials_kwargs = self._to_credential_kwargs(credentials)
+        client = AzureOpenAI(**credentials_kwargs)
+
+        extra_model_kwargs = {}
+        if user:
+            extra_model_kwargs['user'] = user
+
+        extra_model_kwargs['encoding_format'] = 'base64'
+
+        context_size = self._get_context_size(model, credentials)
+        max_chunks = self._get_max_chunks(model, credentials)
+
+        embeddings: list[list[float]] = [[] for _ in range(len(texts))]
+        tokens = []
+        indices = []
+        used_tokens = 0
+
+        try:
+            enc = tiktoken.encoding_for_model(base_model_name)
+        except KeyError:
+            enc = tiktoken.get_encoding("cl100k_base")
+
+        for i, text in enumerate(texts):
+            token = enc.encode(
+                text
+            )
+            for j in range(0, len(token), context_size):
+                tokens += [token[j: j + context_size]]
+                indices += [i]
+
+        batched_embeddings = []
+        _iter = range(0, len(tokens), max_chunks)
+
+        for i in _iter:
+            embeddings, embedding_used_tokens = self._embedding_invoke(
+                model=model,
+                client=client,
+                texts=tokens[i: i + max_chunks],
+                extra_model_kwargs=extra_model_kwargs
+            )
+
+            used_tokens += embedding_used_tokens
+            batched_embeddings += [data for data in embeddings]
+
+        results: list[list[list[float]]] = [[] for _ in range(len(texts))]
+        num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))]
+        for i in range(len(indices)):
+            results[indices[i]].append(batched_embeddings[i])
+            num_tokens_in_batch[indices[i]].append(len(tokens[i]))
+
+        for i in range(len(texts)):
+            _result = results[i]
+            if len(_result) == 0:
+                embeddings, embedding_used_tokens = self._embedding_invoke(
+                    model=model,
+                    client=client,
+                    texts=[""],
+                    extra_model_kwargs=extra_model_kwargs
+                )
+
+                used_tokens += embedding_used_tokens
+                average = embeddings[0]
+            else:
+                average = np.average(_result, axis=0, weights=num_tokens_in_batch[i])
+            embeddings[i] = (average / np.linalg.norm(average)).tolist()
+
+        # calc usage
+        usage = self._calc_response_usage(
+            model=model,
+            credentials=credentials,
+            tokens=used_tokens
+        )
+
+        return TextEmbeddingResult(
+            embeddings=embeddings,
+            usage=usage,
+            model=base_model_name
+        )
+
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        if len(texts) == 0:
+            return 0
+
+        try:
+            enc = tiktoken.encoding_for_model(credentials['base_model_name'])
+        except KeyError:
+            enc = tiktoken.get_encoding("cl100k_base")
+
+        total_num_tokens = 0
+        for text in texts:
+            # calculate the number of tokens in the encoded text
+            tokenized_text = enc.encode(text)
+            total_num_tokens += len(tokenized_text)
+
+        return total_num_tokens
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        if 'openai_api_base' not in credentials:
+            raise CredentialsValidateFailedError('Azure OpenAI API Base Endpoint is required')
+
+        if 'openai_api_key' not in credentials:
+            raise CredentialsValidateFailedError('Azure OpenAI API key is required')
+
+        if 'base_model_name' not in credentials:
+            raise CredentialsValidateFailedError('Base Model Name is required')
+
+        if not self._get_ai_model_entity(credentials['base_model_name'], model):
+            raise CredentialsValidateFailedError(f'Base Model Name {credentials["base_model_name"]} is invalid')
+
+        try:
+            credentials_kwargs = self._to_credential_kwargs(credentials)
+            client = AzureOpenAI(**credentials_kwargs)
+
+            self._embedding_invoke(
+                model=model,
+                client=client,
+                texts=['ping'],
+                extra_model_kwargs={}
+            )
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        ai_model_entity = self._get_ai_model_entity(credentials['base_model_name'], model)
+        return ai_model_entity.entity
+
+    @staticmethod
+    def _embedding_invoke(model: str, client: AzureOpenAI, texts: list[str],
+                          extra_model_kwargs: dict) -> Tuple[list[list[float]], int]:
+        response = client.embeddings.create(
+            input=texts,
+            model=model,
+            **extra_model_kwargs,
+        )
+
+        if 'encoding_format' in extra_model_kwargs and extra_model_kwargs['encoding_format'] == 'base64':
+            # decode base64 embedding
+            return ([list(np.frombuffer(base64.b64decode(data.embedding), dtype="float32")) for data in response.data],
+                    response.usage.total_tokens)
+
+        return [data.embedding for data in response.data], response.usage.total_tokens
+
+    def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
+        input_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=tokens
+        )
+
+        # transform usage
+        usage = EmbeddingUsage(
+            tokens=tokens,
+            total_tokens=tokens,
+            unit_price=input_price_info.unit_price,
+            price_unit=input_price_info.unit,
+            total_price=input_price_info.total_amount,
+            currency=input_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage
+
+    @staticmethod
+    def _get_ai_model_entity(base_model_name: str, model: str) -> AzureBaseModel:
+        for ai_model_entity in EMBEDDING_BASE_MODELS:
+            if ai_model_entity.base_model_name == base_model_name:
+                ai_model_entity.entity.model = model
+                ai_model_entity.entity.label.en_US = model
+                ai_model_entity.entity.label.zh_Hans = model
+                return ai_model_entity
+
+        return None

api/core/model_runtime/model_providers/baichuan/_assets/icon_l_en.svg

@@ -0,0 +1,19 @@
+<svg width="130" height="24" viewBox="0 0 130 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.58154 1.7793H6.52779L4.34655 6.20409V17.7335L1.91602 22.2206H7.21333L9.58154 17.7335V1.7793ZM11.5761 1.7793H16.8111V22.2206H11.5761V1.7793ZM23.9166 1.7793H18.6816V6.01712H23.9166V1.7793ZM23.9166 7.38818H18.6816V22.2206H23.9166V7.38818Z" fill="url(#paint0_radial_11622_96091)"/>
+<path d="M129.722 6.83203V18H127.482V6.83203H129.722Z" fill="#FF6A34"/>
+<path d="M123.196 15.872H118.748L118.012 18H115.66L119.676 6.81604H122.284L126.3 18H123.932L123.196 15.872ZM122.588 14.08L120.972 9.40804L119.356 14.08H122.588Z" fill="#FF6A34"/>
+<path d="M110.962 18H108.722L103.65 10.336V18H101.41V6.81598H103.65L108.722 14.496V6.81598H110.962V18Z" fill="#FF6A34"/>
+<path d="M97.1258 15.872H92.6778L91.9418 18H89.5898L93.6058 6.81604H96.2138L100.23 18H97.8618L97.1258 15.872ZM96.5178 14.08L94.9018 9.40804L93.2858 14.08H96.5178Z" fill="#FF6A34"/>
+<path d="M81.6482 6.83203V13.744C81.6482 14.5014 81.8455 15.0827 82.2402 15.488C82.6349 15.8827 83.1895 16.08 83.9042 16.08C84.6295 16.08 85.1895 15.8827 85.5842 15.488C85.9789 15.0827 86.1762 14.5014 86.1762 13.744V6.83203H88.4322V13.728C88.4322 14.6774 88.2242 15.4827 87.8082 16.144C87.4029 16.7947 86.8535 17.2854 86.1602 17.616C85.4775 17.9467 84.7149 18.112 83.8722 18.112C83.0402 18.112 82.2829 17.9467 81.6002 17.616C80.9282 17.2854 80.3949 16.7947 80.0002 16.144C79.6055 15.4827 79.4082 14.6774 79.4082 13.728V6.83203H81.6482Z" fill="#FF6A34"/>
+<path d="M77.557 6.83203V18H75.317V13.248H70.533V18H68.293V6.83203H70.533V11.424H75.317V6.83203H77.557Z" fill="#FF6A34"/>
+<path d="M55.7871 12.4C55.7871 11.3013 56.0324 10.32 56.5231 9.45599C57.0244 8.58132 57.7018 7.90399 58.5551 7.42399C59.4191 6.93332 60.3844 6.68799 61.4511 6.68799C62.6991 6.68799 63.7924 7.00799 64.7311 7.64799C65.6698 8.28799 66.3258 9.17332 66.6991 10.304H64.1231C63.8671 9.77065 63.5044 9.37065 63.0351 9.10399C62.5764 8.83732 62.0431 8.70399 61.4351 8.70399C60.7844 8.70399 60.2031 8.85865 59.6911 9.16799C59.1898 9.46665 58.7951 9.89332 58.5071 10.448C58.2298 11.0027 58.0911 11.6533 58.0911 12.4C58.0911 13.136 58.2298 13.7867 58.5071 14.352C58.7951 14.9067 59.1898 15.3387 59.6911 15.648C60.2031 15.9467 60.7844 16.096 61.4351 16.096C62.0431 16.096 62.5764 15.9627 63.0351 15.696C63.5044 15.4187 63.8671 15.0133 64.1231 14.48H66.6991C66.3258 15.6213 65.6698 16.512 64.7311 17.152C63.8031 17.7813 62.7098 18.096 61.4511 18.096C60.3844 18.096 59.4191 17.856 58.5551 17.376C57.7018 16.8853 57.0244 16.208 56.5231 15.344C56.0324 14.48 55.7871 13.4987 55.7871 12.4Z" fill="#FF6A34"/>
+<path d="M54.4373 6.83203V18H52.1973V6.83203H54.4373Z" fill="#FF6A34"/>
+<path d="M47.913 15.872H43.465L42.729 18H40.377L44.393 6.81598H47.001L51.017 18H48.649L47.913 15.872ZM47.305 14.08L45.689 9.40798L44.073 14.08H47.305Z" fill="#FF6A34"/>
+<path d="M37.4395 12.272C38.0688 12.3893 38.5862 12.704 38.9915 13.216C39.3968 13.728 39.5995 14.3146 39.5995 14.976C39.5995 15.5733 39.4502 16.1013 39.1515 16.56C38.8635 17.008 38.4422 17.36 37.8875 17.616C37.3328 17.872 36.6768 18 35.9195 18H31.1035V6.83197H35.7115C36.4688 6.83197 37.1195 6.95464 37.6635 7.19997C38.2182 7.4453 38.6342 7.78664 38.9115 8.22397C39.1995 8.6613 39.3435 9.1573 39.3435 9.71197C39.3435 10.3626 39.1675 10.9066 38.8155 11.344C38.4742 11.7813 38.0155 12.0906 37.4395 12.272ZM33.3435 11.44H35.3915C35.9248 11.44 36.3355 11.3226 36.6235 11.088C36.9115 10.8426 37.0555 10.496 37.0555 10.048C37.0555 9.59997 36.9115 9.2533 36.6235 9.00797C36.3355 8.76264 35.9248 8.63997 35.3915 8.63997H33.3435V11.44ZM35.5995 16.176C36.1435 16.176 36.5648 16.048 36.8635 15.792C37.1728 15.536 37.3275 15.1733 37.3275 14.704C37.3275 14.224 37.1675 13.8506 36.8475 13.584C36.5275 13.3066 36.0955 13.168 35.5515 13.168H33.3435V16.176H35.5995Z" fill="#FF6A34"/>
+<defs>
+<radialGradient id="paint0_radial_11622_96091" cx="0" cy="0" r="1" gradientUnits="userSpaceOnUse" gradientTransform="translate(6.5 5.5) rotate(45) scale(20.5061 22.0704)">
+<stop stop-color="#FEBD3F"/>
+<stop offset="0.77608" stop-color="#FF6933"/>
+</radialGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/baichuan/_assets/icon_s_en.svg

@@ -0,0 +1,11 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Baichuan">
+<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M8.58154 1.7793H5.52779L3.34655 6.20409V17.7335L0.916016 22.2206H6.21333L8.58154 17.7335V1.7793ZM10.5761 1.7793H15.8111V22.2206H10.5761V1.7793ZM22.9166 1.7793H17.6816V6.01712H22.9166V1.7793ZM22.9166 7.38818H17.6816V22.2206H22.9166V7.38818Z" fill="url(#paint0_radial_11622_96084)"/>
+</g>
+<defs>
+<radialGradient id="paint0_radial_11622_96084" cx="0" cy="0" r="1" gradientUnits="userSpaceOnUse" gradientTransform="translate(5.5 5.5) rotate(45) scale(20.5061 22.0704)">
+<stop stop-color="#FEBD3F"/>
+<stop offset="0.77608" stop-color="#FF6933"/>
+</radialGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/baichuan/baichuan.py

@@ -0,0 +1,29 @@
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+import logging
+
+logger = logging.getLogger(__name__)
+
+class BaichuanProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.LLM)
+
+            # Use `baichuan2-turbo` model for validate,
+            model_instance.validate_credentials(
+                model='baichuan2-turbo',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/baichuan/baichuan.yaml

@@ -0,0 +1,37 @@
+provider: baichuan
+label:
+  en_US: Baichuan
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#FFF6F2"
+help:
+  title:
+    en_US: Get your API Key from BAICHUAN AI 
+    zh_Hans: 从百川智能获取您的 API Key
+  url:
+    en_US: https://www.baichuan-ai.com
+supported_model_types:
+- llm
+- text-embedding
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: api_key
+    label:
+      en_US: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API Key
+      en_US: Enter your API Key
+  - variable: secret_key
+    label:
+      en_US: Secret Key
+    type: secret-input
+    required: false
+    placeholder:
+      zh_Hans: 在此输入您的 Secret Key
+      en_US: Enter your Secret Key

api/core/model_runtime/model_providers/baichuan/llm/baichuan2-53b.yaml

@@ -0,0 +1,42 @@
+model: baichuan2-53b
+label:
+  en_US: Baichuan2-53B
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 4000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 1000
+  min: 1
+  max: 4000
+- name: presence_penalty
+  use_template: presence_penalty
+- name: frequency_penalty
+  use_template: frequency_penalty
+- name: with_search_enhance
+  label:
+    zh_Hans: 搜索增强
+    en_US: Search Enhance
+  type: boolean
+  help:
+    zh_Hans: 允许模型自行进行外部搜索，以增强生成结果。
+    en_US: Allow the model to perform external search to enhance the generation results.
+  required: false

api/core/model_runtime/model_providers/baichuan/llm/baichuan2-turbo-192k.yaml

@@ -0,0 +1,42 @@
+model: baichuan2-turbo-192k
+label:
+  en_US: Baichuan2-Turbo-192K
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 192000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 8000
+  min: 1
+  max: 192000
+- name: presence_penalty
+  use_template: presence_penalty
+- name: frequency_penalty
+  use_template: frequency_penalty
+- name: with_search_enhance
+  label:
+    zh_Hans: 搜索增强
+    en_US: Search Enhance
+  type: boolean
+  help:
+    zh_Hans: 允许模型自行进行外部搜索，以增强生成结果。
+    en_US: Allow the model to perform external search to enhance the generation results.
+  required: false

api/core/model_runtime/model_providers/baichuan/llm/baichuan2-turbo.yaml

@@ -0,0 +1,42 @@
+model: baichuan2-turbo
+label:
+  en_US: Baichuan2-Turbo
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 192000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 8000
+  min: 1
+  max: 192000
+- name: presence_penalty
+  use_template: presence_penalty
+- name: frequency_penalty
+  use_template: frequency_penalty
+- name: with_search_enhance
+  label:
+    zh_Hans: 搜索增强
+    en_US: Search Enhance
+  type: boolean
+  help:
+    zh_Hans: 允许模型自行进行外部搜索，以增强生成结果。
+    en_US: Allow the model to perform external search to enhance the generation results.
+  required: false

api/core/model_runtime/model_providers/baichuan/llm/baichuan_tokenizer.py

@@ -0,0 +1,19 @@
+import re
+
+class BaichuanTokenizer(object):
+    @classmethod
+    def count_chinese_characters(cls, text: str) -> int:
+        return len(re.findall(r'[\u4e00-\u9fa5]', text))
+
+    @classmethod
+    def count_english_vocabularies(cls, text: str) -> int:
+        # remove all non-alphanumeric characters but keep spaces and other symbols like !, ., etc.
+        text = re.sub(r'[^a-zA-Z0-9\s]', '', text)
+        # count the number of words not characters
+        return len(text.split())
+    
+    @classmethod
+    def _get_num_tokens(cls, text: str) -> int:
+        # tokens = number of Chinese characters + number of English words * 1.3 (for estimation only, subject to actual return)
+        # https://platform.baichuan-ai.com/docs/text-Embedding
+        return int(cls.count_chinese_characters(text) + cls.count_english_vocabularies(text) * 1.3)

api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py

@@ -0,0 +1,199 @@
+from os.path import join
+from typing import List, Optional, Generator, Union, Dict, Any
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import \
+    InsufficientAccountBalance, InvalidAPIKeyError, InternalServerError, RateLimitReachedError, InvalidAuthenticationError, BadRequestError
+from enum import Enum
+from json import dumps, loads
+from requests import post
+from time import time
+from hashlib import md5
+
+class BaichuanMessage:
+    class Role(Enum):
+        USER = 'user'
+        ASSISTANT = 'assistant'
+        # Baichuan does not have system message
+        _SYSTEM = 'system'
+
+    role: str = Role.USER.value
+    content: str
+    usage: Dict[str, int] = None
+    stop_reason: str = ''
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            'role': self.role,
+            'content': self.content,
+        }
+    
+    def __init__(self, content: str, role: str = 'user') -> None:
+        self.content = content
+        self.role = role
+
+class BaichuanModel(object):
+    api_key: str
+    secret_key: str
+
+    def __init__(self, api_key: str, secret_key: str = '') -> None:
+        self.api_key = api_key
+        self.secret_key = secret_key
+
+    def _model_mapping(self, model: str) -> str:
+        return {
+            'baichuan2-turbo': 'Baichuan2-Turbo',
+            'baichuan2-turbo-192k': 'Baichuan2-Turbo-192k',
+            'baichuan2-53b': 'Baichuan2-53B',
+        }[model]
+
+    def _handle_chat_generate_response(self, response) -> BaichuanMessage:
+            resp = response.json()
+            choices = resp.get('choices', [])
+            message = BaichuanMessage(content='', role='assistant')
+            for choice in choices:
+                message.content += choice['message']['content']
+                message.role = choice['message']['role']
+                if choice['finish_reason']:
+                    message.stop_reason = choice['finish_reason']
+
+            if 'usage' in resp:
+                message.usage = {
+                    'prompt_tokens': resp['usage']['prompt_tokens'],
+                    'completion_tokens': resp['usage']['completion_tokens'],
+                    'total_tokens': resp['usage']['total_tokens'],
+                }
+            
+            return message
+    
+    def _handle_chat_stream_generate_response(self, response) -> Generator:
+        for line in response.iter_lines():
+            if not line:
+                continue
+            line = line.decode('utf-8')
+            # remove the first `data: ` prefix
+            if line.startswith('data:'):
+                line = line[5:].strip()
+            try:
+                data = loads(line)
+            except Exception as e:
+                if line.strip() == '[DONE]':
+                    return
+            choices = data.get('choices', [])
+            # save stop reason temporarily
+            stop_reason = ''
+            for choice in choices:
+                if 'finish_reason' in choice and choice['finish_reason']:
+                    stop_reason = choice['finish_reason']
+
+                if len(choice['delta']['content']) == 0:
+                    continue
+                yield BaichuanMessage(**choice['delta'])
+
+            # if there is usage, the response is the last one, yield it and return
+            if 'usage' in data:
+                message = BaichuanMessage(content='', role='assistant')
+                message.usage = {
+                    'prompt_tokens': data['usage']['prompt_tokens'],
+                    'completion_tokens': data['usage']['completion_tokens'],
+                    'total_tokens': data['usage']['total_tokens'],
+                }
+                message.stop_reason = stop_reason
+                yield message
+
+    def _build_parameters(self, model: str, stream: bool, messages: List[BaichuanMessage],
+                               parameters: Dict[str, Any]) \
+        -> Dict[str, Any]:
+        if model == 'baichuan2-turbo' or model == 'baichuan2-turbo-192k' or model == 'baichuan2-53b':
+            prompt_messages = []
+            for message in messages:
+                if message.role == BaichuanMessage.Role.USER.value or message.role == BaichuanMessage.Role._SYSTEM.value:
+                    # check if the latest message is a user message
+                    if len(prompt_messages) > 0 and prompt_messages[-1]['role'] == BaichuanMessage.Role.USER.value:
+                        prompt_messages[-1]['content'] += message.content
+                    else:
+                        prompt_messages.append({
+                            'content': message.content,
+                            'role': BaichuanMessage.Role.USER.value,
+                        })
+                elif message.role == BaichuanMessage.Role.ASSISTANT.value:
+                    prompt_messages.append({
+                        'content': message.content,
+                        'role': message.role,
+                    })
+            # turbo api accepts flat parameters
+            return {
+                'model': self._model_mapping(model),
+                'stream': stream,
+                'messages': prompt_messages,
+                **parameters,
+            }
+        else:
+            raise BadRequestError(f"Unknown model: {model}")
+        
+    def _build_headers(self, model: str, data: Dict[str, Any]) -> Dict[str, Any]:
+        if model == 'baichuan2-turbo' or model == 'baichuan2-turbo-192k' or model == 'baichuan2-53b':
+            # there is no secret key for turbo api
+            return {
+                'Content-Type': 'application/json',
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ',
+                'Authorization': 'Bearer ' + self.api_key,
+            }
+        else:
+            raise BadRequestError(f"Unknown model: {model}")
+        
+    def _calculate_md5(self, input_string):
+        return md5(input_string.encode('utf-8')).hexdigest()
+
+    def generate(self, model: str, stream: bool, messages: List[BaichuanMessage], 
+                 parameters: Dict[str, Any], timeout: int) \
+        -> Union[Generator, BaichuanMessage]:
+        
+        if model == 'baichuan2-turbo' or model == 'baichuan2-turbo-192k' or model == 'baichuan2-53b':
+            api_base = 'https://api.baichuan-ai.com/v1/chat/completions'
+        else:
+            raise BadRequestError(f"Unknown model: {model}")
+        
+        try:
+            data = self._build_parameters(model, stream, messages, parameters)
+            headers = self._build_headers(model, data)
+        except KeyError:
+            raise InternalServerError(f"Failed to build parameters for model: {model}")
+
+        try:
+            response = post(
+                url=api_base,
+                headers=headers,
+                data=dumps(data),
+                timeout=timeout,
+                stream=stream
+            )
+        except Exception as e:
+            raise InternalServerError(f"Failed to invoke model: {e}")
+        
+        if response.status_code != 200:
+            try:
+                resp = response.json()
+                # try to parse error message
+                err = resp['error']['code']
+                msg = resp['error']['message']
+            except Exception as e:
+                raise InternalServerError(f"Failed to convert response to json: {e} with text: {response.text}")
+
+            if err == 'invalid_api_key':
+                raise InvalidAPIKeyError(msg)
+            elif err == 'insufficient_quota':
+                raise InsufficientAccountBalance(msg)
+            elif err == 'invalid_authentication':
+                raise InvalidAuthenticationError(msg)
+            elif 'rate' in err:
+                raise RateLimitReachedError(msg)
+            elif 'internal' in err:
+                raise InternalServerError(msg)
+            elif err == 'api_key_empty':
+                raise InvalidAPIKeyError(msg)
+            else:
+                raise InternalServerError(f"Unknown error: {err} with message: {msg}")
+            
+        if stream:
+            return self._handle_chat_stream_generate_response(response)
+        else:
+            return self._handle_chat_generate_response(response)

api/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo_errors.py

@@ -0,0 +1,17 @@
+class InvalidAuthenticationError(Exception):
+    pass
+
+class InvalidAPIKeyError(Exception):
+    pass
+
+class RateLimitReachedError(Exception):
+    pass
+
+class InsufficientAccountBalance(Exception):
+    pass
+
+class InternalServerError(Exception):
+    pass
+
+class BadRequestError(Exception):
+    pass

api/core/model_runtime/model_providers/baichuan/llm/llm.py

@@ -0,0 +1,194 @@
+from typing import Generator, List, Optional, Union, cast
+from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, UserPromptMessage, SystemPromptMessage
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import \
+    InsufficientAccountBalance, InvalidAPIKeyError, InternalServerError, RateLimitReachedError, InvalidAuthenticationError, BadRequestError
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo import BaichuanModel, BaichuanMessage
+from core.model_runtime.model_providers.baichuan.llm.baichuan_tokenizer import BaichuanTokenizer
+
+class BaichuanLarguageModel(LargeLanguageModel):
+    def _invoke(self, model: str, credentials: dict, 
+                prompt_messages: list[PromptMessage], model_parameters: dict, 
+                tools: list[PromptMessageTool] | None = None, stop: List[str] | None = None, 
+                stream: bool = True, user: str | None = None) \
+            -> LLMResult | Generator:
+        return self._generate(model=model, credentials=credentials, prompt_messages=prompt_messages,
+                                model_parameters=model_parameters, tools=tools, stop=stop, stream=stream, user=user)
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: list[PromptMessageTool] | None = None) -> int:
+        return self._num_tokens_from_messages(prompt_messages)
+
+    def _num_tokens_from_messages(self, messages: List[PromptMessage],) -> int:
+        """Calculate num tokens for baichuan model"""
+        def tokens(text: str):
+            return BaichuanTokenizer._get_num_tokens(text)
+
+        tokens_per_message = 3
+
+        num_tokens = 0
+        messages_dict = [self._convert_prompt_message_to_dict(m) for m in messages]
+        for message in messages_dict:
+            num_tokens += tokens_per_message
+            for key, value in message.items():
+                if isinstance(value, list):
+                    text = ''
+                    for item in value:
+                        if isinstance(item, dict) and item['type'] == 'text':
+                            text += item['text']
+
+                    value = text
+
+                num_tokens += tokens(str(value))
+        num_tokens += 3
+
+        return num_tokens
+
+    def _convert_prompt_message_to_dict(self, message: PromptMessage) -> dict:
+        """
+        Convert PromptMessage to dict for Baichuan
+        """
+        if isinstance(message, UserPromptMessage):
+            message = cast(UserPromptMessage, message)
+            if isinstance(message.content, str):
+                message_dict = {"role": "user", "content": message.content}
+            else:
+                raise ValueError("User message content must be str")
+        elif isinstance(message, AssistantPromptMessage):
+            message = cast(AssistantPromptMessage, message)
+            message_dict = {"role": "assistant", "content": message.content}
+        elif isinstance(message, SystemPromptMessage):
+            message = cast(SystemPromptMessage, message)
+            message_dict = {"role": "user", "content": message.content}
+        else:
+            raise ValueError(f"Unknown message type {type(message)}")
+        
+        return message_dict
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        # ping
+        instance = BaichuanModel(
+            api_key=credentials['api_key'],
+            secret_key=credentials.get('secret_key', '')
+        )
+
+        try:
+            instance.generate(model=model, stream=False, messages=[
+                BaichuanMessage(content='ping', role='user')
+            ], parameters={}, timeout=10)
+        except InvalidAPIKeyError as e:
+            raise CredentialsValidateFailedError(f"Invalid API key: {e}")
+
+    def _generate(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], 
+                 model_parameters: dict, tools: list[PromptMessageTool] | None = None, 
+                 stop: List[str] | None = None, stream: bool = True, user: str | None = None) \
+            -> LLMResult | Generator:
+        if tools is not None and len(tools) > 0:
+            raise InvokeBadRequestError(f"Baichuan model doesn't support tools")
+        
+        instance = BaichuanModel(
+            api_key=credentials['api_key'],
+            secret_key=credentials.get('secret_key', '')
+        )
+
+        # convert prompt messages to baichuan messages
+        messages = [
+            BaichuanMessage(
+                content=message.content if isinstance(message.content, str) else ''.join([
+                    content.data for content in message.content
+                ]),
+                role=message.role.value
+            ) for message in prompt_messages
+        ]
+
+        # invoke model
+        response = instance.generate(model=model, stream=stream, messages=messages, parameters=model_parameters, timeout=60)
+
+        if stream:
+            return self._handle_chat_generate_stream_response(model, prompt_messages, credentials, response)
+        
+        return self._handle_chat_generate_response(model, prompt_messages, credentials, response)
+
+    def _handle_chat_generate_response(self, model: str,
+                                       prompt_messages: list[PromptMessage],
+                                       credentials: dict,
+                                       response: BaichuanMessage) -> LLMResult:
+        # convert baichuan message to llm result
+        usage = self._calc_response_usage(model=model, credentials=credentials, prompt_tokens=response.usage['prompt_tokens'], completion_tokens=response.usage['completion_tokens'])
+        return LLMResult(
+            model=model,
+            prompt_messages=prompt_messages,
+            message=AssistantPromptMessage(
+                content=response.content,
+                tool_calls=[]
+            ),
+            usage=usage,
+        )
+
+    def _handle_chat_generate_stream_response(self, model: str,
+                                              prompt_messages: list[PromptMessage],
+                                              credentials: dict,
+                                              response: Generator[BaichuanMessage, None, None]) -> Generator:
+        for message in response:
+            if message.usage:
+                usage = self._calc_response_usage(model=model, credentials=credentials, prompt_tokens=message.usage['prompt_tokens'], completion_tokens=message.usage['completion_tokens'])
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=0,
+                        message=AssistantPromptMessage(
+                            content=message.content,
+                            tool_calls=[]
+                        ),
+                        usage=usage,
+                        finish_reason=message.stop_reason if message.stop_reason else None,
+                    ),
+                )
+            else:
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=0,
+                        message=AssistantPromptMessage(
+                            content=message.content,
+                            tool_calls=[]
+                        ),
+                        finish_reason=message.stop_reason if message.stop_reason else None,
+                    ),
+                )
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        return {
+            InvokeConnectionError: [
+            ],
+            InvokeServerUnavailableError: [
+                InternalServerError
+            ],
+            InvokeRateLimitError: [
+                RateLimitReachedError
+            ],
+            InvokeAuthorizationError: [
+                InvalidAuthenticationError,
+                InsufficientAccountBalance,
+                InvalidAPIKeyError,
+            ],
+            InvokeBadRequestError: [
+                BadRequestError,
+                KeyError
+            ]
+        }

api/core/model_runtime/model_providers/baichuan/text_embedding/baichuan-text-embedding.yaml

@@ -0,0 +1,5 @@
+model: baichuan-text-embedding
+model_type: text-embedding
+model_properties:
+  context_size: 512
+  max_chunks: 16

api/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py

@@ -0,0 +1,178 @@
+from typing import Optional
+
+from core.model_runtime.entities.model_entities import PriceType
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
+from core.model_runtime.errors.invoke import InvokeError, InvokeConnectionError, InvokeServerUnavailableError, \
+    InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError
+from core.model_runtime.model_providers.baichuan.llm.baichuan_tokenizer import BaichuanTokenizer
+from core.model_runtime.model_providers.baichuan.llm.baichuan_turbo_errors import InvalidAPIKeyError, InsufficientAccountBalance, \
+    InvalidAuthenticationError, RateLimitReachedError, InternalServerError, BadRequestError
+
+from requests import post
+from json import dumps, loads
+
+import time
+
+class BaichuanTextEmbeddingModel(TextEmbeddingModel):
+    """
+    Model class for BaiChuan text embedding model.
+    """
+    api_base: str = 'http://api.baichuan-ai.com/v1/embeddings'
+
+    def _invoke(self, model: str, credentials: dict,
+                texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        """
+        Invoke text embedding model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :param user: unique user id
+        :return: embeddings result
+        """
+        api_key = credentials['api_key']
+        if model != 'baichuan-text-embedding':
+            raise ValueError('Invalid model name')
+        if not api_key:
+            raise CredentialsValidateFailedError('api_key is required')
+        url = self.api_base
+        headers = {
+            'Authorization': 'Bearer ' + api_key,
+            'Content-Type': 'application/json'
+        }
+
+        data = {
+            'model': 'Baichuan-Text-Embedding',
+            'input': texts
+        }
+
+        try:
+            response = post(url, headers=headers, data=dumps(data))
+        except Exception as e:
+            raise InvokeConnectionError(e)
+        
+        if response.status_code != 200:
+            try:
+                resp = response.json()
+                # try to parse error message
+                err = resp['error']['code']
+                msg = resp['error']['message']
+            except Exception as e:
+                raise InternalServerError(f"Failed to convert response to json: {e} with text: {response.text}")
+
+            if err == 'invalid_api_key':
+                raise InvalidAPIKeyError(msg)
+            elif err == 'insufficient_quota':
+                raise InsufficientAccountBalance(msg)
+            elif err == 'invalid_authentication':
+                raise InvalidAuthenticationError(msg)
+            elif 'rate' in err:
+                raise RateLimitReachedError(msg)
+            elif 'internal' in err:
+                raise InternalServerError(msg)
+            elif err == 'api_key_empty':
+                raise InvalidAPIKeyError(msg)
+            else:
+                raise InternalServerError(f"Unknown error: {err} with message: {msg}")
+            
+        try:
+            resp = response.json()
+            embeddings = resp['data']
+            usage = resp['usage']
+        except Exception as e:
+            raise InternalServerError(f"Failed to convert response to json: {e} with text: {response.text}")
+
+        usage = self._calc_response_usage(model=model, credentials=credentials, tokens=usage['total_tokens'])
+
+        result = TextEmbeddingResult(
+            model=model,
+            embeddings=[[
+                float(data) for data in x['embedding']
+            ] for x in embeddings],
+            usage=usage
+        )
+
+        return result
+
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :return:
+        """
+        num_tokens = 0
+        for text in texts:
+            # use BaichuanTokenizer to get num tokens
+            num_tokens += BaichuanTokenizer._get_num_tokens(text)
+        return num_tokens
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._invoke(model=model, credentials=credentials, texts=['ping'])
+        except InvalidAPIKeyError:
+            raise CredentialsValidateFailedError('Invalid api key')
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        return {
+            InvokeConnectionError: [
+            ],
+            InvokeServerUnavailableError: [
+                InternalServerError
+            ],
+            InvokeRateLimitError: [
+                RateLimitReachedError
+            ],
+            InvokeAuthorizationError: [
+                InvalidAuthenticationError,
+                InsufficientAccountBalance,
+                InvalidAPIKeyError,
+            ],
+            InvokeBadRequestError: [
+                BadRequestError,
+                KeyError
+            ]
+        }
+    
+    def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
+        """
+        Calculate response usage
+
+        :param model: model name
+        :param credentials: model credentials
+        :param tokens: input tokens
+        :return: usage
+        """
+        # get input price info
+        input_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=tokens
+        )
+
+        # transform usage
+        usage = EmbeddingUsage(
+            tokens=tokens,
+            total_tokens=tokens,
+            unit_price=input_price_info.unit_price,
+            price_unit=input_price_info.unit,
+            total_price=input_price_info.total_amount,
+            currency=input_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage

api/core/model_runtime/model_providers/chatglm/_assets/icon_s_en.svg

@@ -0,0 +1,9 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<mask id="mask0_8587_60212" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="1" y="2" width="23" height="21">
+<path d="M23.8 2H1V22.4H23.8V2Z" fill="white"/>
+</mask>
+<g mask="url(#mask0_8587_60212)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M3.86378 14.4544C3.86378 13.0981 4.67438 11.737 6.25923 10.6634C7.83827 9.59364 10.0864 8.89368 12.6282 8.89368C15.17 8.89368 17.4182 9.59364 18.9972 10.6634C19.7966 11.2049 20.399 11.8196 20.7998 12.4699C21.2873 11.5802 21.4969 10.6351 21.3835 9.69252C21.3759 9.62928 21.3824 9.56766 21.4005 9.5106C21.0758 9.21852 20.7259 8.94624 20.3558 8.69556C18.3272 7.32126 15.5915 6.50964 12.6282 6.50964C9.66497 6.50964 6.92918 7.32126 4.90058 8.69556C2.8778 10.0659 1.45703 12.0812 1.45703 14.4544C1.45703 16.8275 2.8778 18.8428 4.90058 20.2132C6.92918 21.5875 9.66497 22.3991 12.6282 22.3991C15.5915 22.3991 18.3272 21.5875 20.3558 20.2132C22.3786 18.8428 23.7994 16.8275 23.7994 14.4544C23.7994 12.9455 23.225 11.5813 22.2868 10.4355C22.2377 11.4917 21.8621 12.5072 21.238 13.43C21.3409 13.7686 21.3926 14.1116 21.3926 14.4544C21.3926 15.8107 20.582 17.1717 18.9972 18.2453C17.4182 19.3151 15.17 20.015 12.6282 20.015C10.0864 20.015 7.83827 19.3151 6.25923 18.2453C4.67438 17.1717 3.86378 15.8107 3.86378 14.4544Z" fill="#3762FF"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M3.84445 11.6838C3.20239 13.4885 3.35368 15.1156 4.18868 16.2838C5.02368 17.452 6.52281 18.1339 8.45459 18.1334C10.3826 18.133 12.6296 17.44 14.6939 15.9922C16.7581 14.5444 18.1643 12.6753 18.8052 10.8739C19.4473 9.0692 19.2959 7.44206 18.461 6.27392C17.626 5.10572 16.1269 4.42389 14.1951 4.42431C12.267 4.42475 10.0201 5.11774 7.95575 6.56552C5.89152 8.01332 4.48529 9.8825 3.84445 11.6838ZM1.53559 10.8778C2.36374 8.55002 4.11254 6.28976 6.54117 4.58645C8.96981 2.88312 11.7029 1.99995 14.1945 1.99939C16.6825 1.99884 19.0426 2.8912 20.4589 4.87263C21.8752 6.85406 21.941 9.35564 21.1141 11.6799C20.2859 14.0077 18.5371 16.2679 16.1085 17.9713C13.6798 19.6746 10.9468 20.5578 8.45513 20.5584C5.9672 20.5589 3.60706 19.6665 2.19075 17.6851C0.774446 15.7036 0.708677 13.2021 1.53559 10.8778Z" fill="#1041F3"/>
+</g>
+</svg>

api/core/model_runtime/model_providers/chatglm/chatglm.py

@@ -0,0 +1,31 @@
+import logging
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class ChatGLMProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.LLM)
+
+            # Use `chatglm3-6b` model for validate,
+            model_instance.validate_credentials(
+                model='chatglm3-6b',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/chatglm/chatglm.yaml

@@ -0,0 +1,28 @@
+provider: chatglm
+label:
+  en_US: ChatGLM
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#F4F7FF"
+help:
+  title:
+    en_US: Deploy ChatGLM to your local
+    zh_Hans: 部署您的本地 ChatGLM
+  url:
+    en_US: https://github.com/THUDM/ChatGLM3
+supported_model_types:
+- llm
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: api_base
+    label:
+      en_US: API URL
+    type: text-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API URL
+      en_US: Enter your API URL

api/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b-32k.yaml

@@ -0,0 +1,21 @@
+model: chatglm2-6b-32k
+label:
+  en_US: ChatGLM2-6B-32K
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 2000
+  min: 1
+  max: 32000

api/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b.yaml

@@ -0,0 +1,21 @@
+model: chatglm2-6b
+label:
+  en_US: ChatGLM2-6B
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 2000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 256
+  min: 1
+  max: 2000

api/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b-32k.yaml

@@ -0,0 +1,22 @@
+model: chatglm3-6b-32k
+label:
+  en_US: ChatGLM3-6B-32K
+model_type: llm
+features:
+- tool-call
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 32000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 8000
+  min: 1
+  max: 32000

api/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b.yaml

@@ -0,0 +1,22 @@
+model: chatglm3-6b
+label:
+  en_US: ChatGLM3-6B
+model_type: llm
+features:
+- tool-call
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 8000
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+  required: false
+- name: max_tokens
+  use_template: max_tokens
+  required: true
+  default: 256
+  min: 1
+  max: 8000

api/core/model_runtime/model_providers/chatglm/llm/llm.py

@@ -0,0 +1,469 @@
+from typing import Generator, List, Optional
+from requests import post
+
+
+from os.path import join
+from typing import cast
+from json import dumps
+
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
+    SystemPromptMessage
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
+    LLMResultChunkDelta
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \
+    PromptMessageFunction, UserPromptMessage, SystemPromptMessage
+from core.model_runtime.utils import helper
+from openai import OpenAI, Stream, \
+    APIConnectionError, APITimeoutError, AuthenticationError, InternalServerError, \
+    RateLimitError, ConflictError, NotFoundError, UnprocessableEntityError, PermissionDeniedError
+from openai.types.chat import ChatCompletionChunk, ChatCompletion
+from openai.types.chat.chat_completion_message import FunctionCall
+from httpx import Timeout
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+class ChatGLMLargeLanguageModel(LargeLanguageModel):
+    def _invoke(self, model: str, credentials: dict, 
+                prompt_messages: list[PromptMessage], model_parameters: dict, 
+                tools: list[PromptMessageTool] | None = None, stop: List[str] | None = None, 
+                stream: bool = True, user: str | None = None) \
+            -> LLMResult | Generator:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        # invoke model
+        return self._generate(
+            model=model,
+            credentials=credentials,
+            prompt_messages=prompt_messages,
+            model_parameters=model_parameters,
+            tools=tools,
+            stop=stop,
+            stream=stream,
+            user=user
+        )
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: list[PromptMessageTool] | None = None) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param tools: tools for tool calling
+        :return:
+        """
+        return self._num_tokens_from_messages(messages=prompt_messages, tools=tools)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._invoke(model=model, credentials=credentials, prompt_messages=[
+                UserPromptMessage(content="ping"),
+            ], model_parameters={
+                "max_tokens": 16,
+            })
+        except Exception as e:
+            raise CredentialsValidateFailedError(str(e))
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        return {
+            InvokeConnectionError: [
+                APIConnectionError,
+                APITimeoutError,
+            ],
+            InvokeServerUnavailableError: [
+                InternalServerError,
+                ConflictError,
+                NotFoundError,
+                UnprocessableEntityError,
+                PermissionDeniedError
+            ],
+            InvokeRateLimitError: [
+                RateLimitError
+            ],
+            InvokeAuthorizationError: [
+                AuthenticationError
+            ],
+            InvokeBadRequestError: [
+                ValueError
+            ]
+        }
+
+    def _generate(self, model: str, credentials: dict, 
+                prompt_messages: list[PromptMessage], model_parameters: dict, 
+                tools: list[PromptMessageTool] | None = None, stop: List[str] | None = None, 
+                stream: bool = True, user: str | None = None) \
+            -> LLMResult | Generator:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: credentials kwargs
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+
+        self._check_chatglm_parameters(model=model, model_parameters=model_parameters, tools=tools)
+
+        kwargs = self._to_client_kwargs(credentials)
+        # init model client
+        client = OpenAI(**kwargs)
+
+        extra_model_kwargs = {}
+        if stop:
+            extra_model_kwargs['stop'] = stop
+
+        if user:
+            extra_model_kwargs['user'] = user
+
+        if tools and len(tools) > 0:
+            extra_model_kwargs['functions'] = [
+                helper.dump_model(tool) for tool in tools
+            ]
+
+        result = client.chat.completions.create(
+            messages=[self._convert_prompt_message_to_dict(m) for m in prompt_messages],
+            model=model,
+            stream=stream,
+            **model_parameters,
+            **extra_model_kwargs
+        )
+
+        if stream:
+            return self._handle_chat_generate_stream_response(
+                model=model, credentials=credentials, response=result, tools=tools, 
+                prompt_messages=prompt_messages
+            )
+        
+        return self._handle_chat_generate_response(
+            model=model, credentials=credentials, response=result, tools=tools, 
+            prompt_messages=prompt_messages
+        )
+        
+    def _check_chatglm_parameters(self, model: str, model_parameters: dict, tools: list[PromptMessageTool]) -> None:
+        if model.find("chatglm2") != -1 and tools is not None and len(tools) > 0:
+            raise InvokeBadRequestError("ChatGLM2 does not support function calling")
+
+    def _convert_prompt_message_to_dict(self, message: PromptMessage) -> dict:
+        """
+        Convert PromptMessage to dict for OpenAI Compatibility API
+        """
+        if isinstance(message, UserPromptMessage):
+            message = cast(UserPromptMessage, message)
+            if isinstance(message.content, str):
+                message_dict = {"role": "user", "content": message.content}
+            else:
+                raise ValueError("User message content must be str")
+        elif isinstance(message, AssistantPromptMessage):
+            message = cast(AssistantPromptMessage, message)
+            message_dict = {"role": "assistant", "content": message.content}
+            if message.tool_calls and len(message.tool_calls) > 0:
+                message_dict["function_call"] = {
+                    "name": message.tool_calls[0].function.name,
+                    "arguments": message.tool_calls[0].function.arguments
+                }
+        elif isinstance(message, SystemPromptMessage):
+            message = cast(SystemPromptMessage, message)
+            message_dict = {"role": "system", "content": message.content}
+        else:
+            raise ValueError(f"Unknown message type {type(message)}")
+        
+        return message_dict
+    
+    def _extract_response_tool_calls(self,
+                                     response_function_calls: list[FunctionCall]) \
+            -> list[AssistantPromptMessage.ToolCall]:
+        """
+        Extract tool calls from response
+
+        :param response_tool_calls: response tool calls
+        :return: list of tool calls
+        """
+        tool_calls = []
+        if response_function_calls:
+            for response_tool_call in response_function_calls:
+                function = AssistantPromptMessage.ToolCall.ToolCallFunction(
+                    name=response_tool_call.name,
+                    arguments=response_tool_call.arguments
+                )
+
+                tool_call = AssistantPromptMessage.ToolCall(
+                    id=0,
+                    type='function',
+                    function=function
+                )
+                tool_calls.append(tool_call)
+
+        return tool_calls
+        
+    def _to_client_kwargs(self, credentials: dict) -> dict:
+        """
+        Convert invoke kwargs to client kwargs
+
+        :param stream: is stream response
+        :param model_name: model name
+        :param credentials: credentials dict
+        :param model_parameters: model parameters
+        :return: client kwargs
+        """
+        client_kwargs = {
+            "timeout": Timeout(315.0, read=300.0, write=10.0, connect=5.0),
+            "api_key": "1",
+            "base_url": join(credentials['api_base'], 'v1')
+        }
+
+        return client_kwargs
+    
+    def _handle_chat_generate_stream_response(self, model: str, credentials: dict, response: Stream[ChatCompletionChunk],
+                                              prompt_messages: list[PromptMessage],
+                                              tools: Optional[list[PromptMessageTool]] = None) \
+            -> Generator:
+        
+        full_response = ''
+
+        for chunk in response:
+            if len(chunk.choices) == 0:
+                continue
+
+            delta = chunk.choices[0]
+
+            if delta.finish_reason is None and (delta.delta.content is None or delta.delta.content == ''):
+                continue
+            
+            # check if there is a tool call in the response
+            function_calls = None
+            if delta.delta.function_call:
+                function_calls = [delta.delta.function_call]
+
+            assistant_message_tool_calls = self._extract_response_tool_calls(function_calls if function_calls else [])
+
+            # transform assistant message to prompt message
+            assistant_prompt_message = AssistantPromptMessage(
+                content=delta.delta.content if delta.delta.content else '',
+                tool_calls=assistant_message_tool_calls
+            )
+
+            if delta.finish_reason is not None:
+                # temp_assistant_prompt_message is used to calculate usage
+                temp_assistant_prompt_message = AssistantPromptMessage(
+                    content=full_response,
+                    tool_calls=assistant_message_tool_calls
+                )
+
+                prompt_tokens = self._num_tokens_from_messages(messages=prompt_messages, tools=tools)
+                completion_tokens = self._num_tokens_from_messages(messages=[temp_assistant_prompt_message], tools=[])
+
+                usage = self._calc_response_usage(model=model, credentials=credentials, 
+                                                  prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
+                
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                        finish_reason=delta.finish_reason,
+                        usage=usage
+                    ),
+                )
+            else:
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    system_fingerprint=chunk.system_fingerprint,
+                    delta=LLMResultChunkDelta(
+                        index=delta.index,
+                        message=assistant_prompt_message,
+                    ),
+                )
+
+                full_response += delta.delta.content
+        
+    def _handle_chat_generate_response(self, model: str, credentials: dict, response: ChatCompletion,
+                                       prompt_messages: list[PromptMessage],
+                                       tools: Optional[list[PromptMessageTool]] = None) \
+            -> LLMResult:
+        """
+        Handle llm chat response
+
+        :param model: model name
+        :param credentials: credentials
+        :param response: response
+        :param prompt_messages: prompt messages
+        :param tools: tools for tool calling
+        :return: llm response
+        """
+        if len(response.choices) == 0:
+            raise InvokeServerUnavailableError("Empty response")
+        assistant_message = response.choices[0].message
+
+        # convert function call to tool call
+        function_calls = assistant_message.function_call
+        tool_calls = self._extract_response_tool_calls([function_calls] if function_calls else [])
+
+        # transform assistant message to prompt message
+        assistant_prompt_message = AssistantPromptMessage(
+            content=assistant_message.content,
+            tool_calls=tool_calls
+        )
+
+        prompt_tokens = self._num_tokens_from_messages(messages=prompt_messages, tools=tools)
+        completion_tokens = self._num_tokens_from_messages(messages=[assistant_prompt_message], tools=tools)
+
+        usage = self._calc_response_usage(model=model, credentials=credentials, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
+
+        response = LLMResult(
+            model=model,
+            prompt_messages=prompt_messages,
+            system_fingerprint=response.system_fingerprint,
+            usage=usage,
+            message=assistant_prompt_message,
+        )
+
+        return response
+    
+    def _num_tokens_from_string(self, text: str, tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """
+        Calculate num tokens for text completion model with tiktoken package.
+
+        :param model: model name
+        :param text: prompt text
+        :param tools: tools for tool calling
+        :return: number of tokens
+        """
+        num_tokens = self._get_num_tokens_by_gpt2(text)
+
+        if tools:
+            num_tokens += self._num_tokens_for_tools(tools)
+
+        return num_tokens
+
+    def _num_tokens_from_messages(self, messages: List[PromptMessage],
+                                  tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """Calculate num tokens for chatglm2 and chatglm3 with GPT2 tokenizer.
+
+        it's too complex to calculate num tokens for chatglm2 and chatglm3 with ChatGLM tokenizer,
+        As a temporary solution we use GPT2 tokenizer instead.
+
+        """
+        def tokens(text: str):
+            return self._get_num_tokens_by_gpt2(text)
+        
+        tokens_per_message = 3
+        tokens_per_name = 1
+        num_tokens = 0
+        messages_dict = [self._convert_prompt_message_to_dict(m) for m in messages]
+        for message in messages_dict:
+            num_tokens += tokens_per_message
+            for key, value in message.items():
+                if isinstance(value, list):
+                    text = ''
+                    for item in value:
+                        if isinstance(item, dict) and item['type'] == 'text':
+                            text += item['text']
+                    value = text
+
+                if key == "function_call":
+                    for t_key, t_value in value.items():
+                        num_tokens += tokens(t_key)
+                        if t_key == "function":
+                            for f_key, f_value in t_value.items():
+                                num_tokens += tokens(f_key)
+                                num_tokens += tokens(f_value)
+                        else:
+                            num_tokens += tokens(t_key)
+                            num_tokens += tokens(t_value)
+                else:
+                    num_tokens += tokens(str(value))
+
+                if key == "name":
+                    num_tokens += tokens_per_name
+
+        # every reply is primed with <im_start>assistant
+        num_tokens += 3
+
+        if tools:
+            num_tokens += self._num_tokens_for_tools(tools)
+
+        return num_tokens
+
+    def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int:
+        """
+        Calculate num tokens for tool calling
+
+        :param encoding: encoding
+        :param tools: tools for tool calling
+        :return: number of tokens
+        """
+        def tokens(text: str):
+            return self._get_num_tokens_by_gpt2(text)
+
+        num_tokens = 0
+        for tool in tools:
+            # calculate num tokens for function object
+            num_tokens += tokens('name')
+            num_tokens += tokens(tool.name)
+            num_tokens += tokens('description')
+            num_tokens += tokens(tool.description)
+            parameters = tool.parameters
+            num_tokens += tokens('parameters')
+            num_tokens += tokens('type')
+            num_tokens += tokens(parameters.get("type"))
+            if 'properties' in parameters:
+                num_tokens += tokens('properties')
+                for key, value in parameters.get('properties').items():
+                    num_tokens += tokens(key)
+                    for field_key, field_value in value.items():
+                        num_tokens += tokens(field_key)
+                        if field_key == 'enum':
+                            for enum_field in field_value:
+                                num_tokens += 3
+                                num_tokens += tokens(enum_field)
+                        else:
+                            num_tokens += tokens(field_key)
+                            num_tokens += tokens(str(field_value))
+            if 'required' in parameters:
+                num_tokens += tokens('required')
+                for required_field in parameters['required']:
+                    num_tokens += 3
+                    num_tokens += tokens(required_field)
+
+        return num_tokens

api/core/model_runtime/model_providers/cohere/_assets/icon_l_en.svg

@@ -0,0 +1,11 @@
+<svg width="120" height="24" viewBox="0 0 120 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M34.4917 21.9129C37.4378 21.9129 40.0162 20.4398 41.0355 17.4656C41.2334 16.8701 40.9496 16.4743 40.384 16.4743H39.2787C38.7689 16.4743 38.4292 16.7002 38.2013 17.1818C37.3239 18.9108 36.1047 19.5324 34.5757 19.5324C31.8553 19.5324 30.1844 17.6335 30.1844 14.4616C30.1844 11.2896 31.9133 9.39083 34.5177 9.39083C36.1046 9.39083 37.4079 10.0704 38.2293 11.6854C38.4852 12.1671 38.795 12.3929 39.3067 12.3929H40.412C40.9776 12.3929 41.2614 12.0251 41.0635 11.4855C39.8742 8.25556 37.2099 7.01035 34.4917 7.01035C30.3843 7.01035 27.3242 10.0424 27.3242 14.4616C27.3242 18.8808 30.2424 21.9129 34.4917 21.9129ZM108.627 13.1584C108.995 10.75 110.638 9.24892 112.876 9.24892C115.115 9.24892 116.786 10.7779 116.983 13.1584H108.627ZM112.99 21.9129C115.596 21.9129 118.203 20.6956 119.478 17.9474C119.79 17.2958 119.506 16.8421 118.94 16.8421H117.892C117.383 16.8421 117.071 17.0679 116.816 17.5216C115.966 19.0227 114.493 19.6463 112.992 19.6463C110.414 19.6463 108.743 17.8894 108.545 15.0292H118.943C119.508 15.0292 119.878 14.7174 119.878 14.1219C119.764 9.67465 116.876 7.01235 112.88 7.01235C108.885 7.01235 105.713 9.90251 105.713 14.4636C105.713 19.0247 108.801 21.9148 112.994 21.9148L112.99 21.9129ZM96.5025 14.8313H97.4378C98.0035 14.8313 98.3152 14.5196 98.4012 13.9239C98.9409 10.0964 101.182 9.5887 103.564 9.70264C104.074 9.72661 104.491 9.33487 104.491 8.82319V7.94575C104.491 7.38012 104.208 7.03833 103.642 7.01035C101.533 6.9304 99.6525 7.65393 98.5651 9.70264C98.5052 9.81455 98.3373 9.78458 98.3233 9.65866L98.1474 8.11365C98.0915 7.54801 97.7796 7.26418 97.212 7.26418H92.9347C92.435 7.26418 92.0272 7.66993 92.0272 8.17161V8.6533C92.0272 9.15298 92.433 9.56072 92.9347 9.56072H94.6916C95.1912 9.56072 95.599 9.96646 95.599 10.4681V13.9239C95.599 14.4236 96.0048 14.8313 96.5064 14.8313H96.5025ZM92.6788 21.631H101.545C102.111 21.631 102.453 21.2913 102.453 20.7236V20.2418C102.453 19.6762 102.113 19.3345 101.545 19.3345H99.2787C98.7131 19.3345 98.3712 18.9947 98.3712 18.4271V16.8681C98.3712 16.3024 98.0315 15.9606 97.4638 15.9606H96.5005C95.9348 15.9606 95.593 16.3004 95.593 16.8681V18.4271C95.593 18.9927 95.2532 19.3345 94.6856 19.3345H92.6749C92.1092 19.3345 91.7674 19.6743 91.7674 20.2418V20.7236C91.7674 21.2893 92.1073 21.631 92.6749 21.631H92.6788ZM78.9955 13.1604C79.3633 10.752 81.0062 9.25092 83.2449 9.25092C85.4834 9.25092 87.1544 10.7799 87.3522 13.1604H78.9955ZM83.3587 21.9148C85.9651 21.9148 88.5714 20.6977 89.8466 17.9493C90.1585 17.2978 89.8746 16.844 89.309 16.844H88.2617C87.7519 16.844 87.4402 17.0699 87.1844 17.5236C86.3349 19.0247 84.8618 19.6482 83.3607 19.6482C80.7824 19.6482 79.1115 17.8914 78.9136 15.0313H89.311C89.8766 15.0313 90.2464 14.7194 90.2464 14.1238C90.1324 9.67665 87.2443 7.01434 83.2488 7.01434C79.2533 7.01434 76.0814 9.9045 76.0814 14.4656C76.0814 19.0266 79.1694 21.9168 83.3628 21.9168L83.3587 21.9148ZM50.5835 21.9148C54.8329 21.9148 57.8649 18.7708 57.8649 14.4636C57.8649 10.1563 54.8329 7.01235 50.5835 7.01235C46.3342 7.01235 43.3022 10.2143 43.3022 14.4636C43.3022 15.455 43.472 16.5602 43.9816 17.7775C44.2375 18.3731 44.7192 18.4571 45.2289 18.0892L46.0504 17.4936C46.4761 17.1818 46.588 16.8141 46.4461 16.2765C46.2202 15.5689 46.1623 14.9453 46.1623 14.4076C46.1623 11.4335 47.9472 9.39283 50.5815 9.39283C53.2159 9.39283 55.0007 11.4035 55.0007 14.4636C55.0007 17.5236 53.2439 19.5344 50.6375 19.5344C49.7301 19.5344 48.8806 19.3645 47.8612 18.5989C47.4355 18.2592 47.0397 18.2032 46.586 18.5429L45.9624 18.9967C45.4527 19.3645 45.3968 19.8741 45.8764 20.2718C47.3496 21.4611 49.0485 21.9148 50.5795 21.9148H50.5835ZM61.4606 21.631H62.3961C62.8957 21.631 63.3035 21.2252 63.3035 20.7236V13.9539C63.3035 11.0937 64.8324 9.39283 67.213 9.39283C69.3656 9.39283 70.6128 10.8099 70.6128 13.4163V20.7255C70.6128 21.2252 71.0186 21.633 71.5203 21.633H72.4836C72.9833 21.633 73.391 21.2272 73.391 20.7255V12.9625C73.391 9.13899 71.4363 7.01434 68.1224 7.01434C65.8659 7.01434 64.5327 7.93776 63.5373 9.22294C63.4613 9.32088 63.3075 9.26691 63.3075 9.14499V2.99092C63.3014 2.48924 62.8957 2.0835 62.3961 2.0835H61.4606C60.9609 2.0835 60.5532 2.48924 60.5532 2.99092V20.7236C60.5532 21.2232 60.959 21.631 61.4606 21.631Z" fill="#39594D"/>
+<mask id="mask0_13223_52628" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="1" y="2" width="20" height="20">
+<path d="M20.8354 2.08319H1.00195V21.9165H20.8354V2.08319Z" fill="white"/>
+</mask>
+<g mask="url(#mask0_13223_52628)">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M7.42768 13.8921C7.96151 13.8921 9.02342 13.8628 10.4912 13.2585C12.2017 12.5542 15.6047 11.2758 18.0597 9.96274C19.7765 9.04432 20.5291 7.82964 20.5291 6.19387C20.5291 3.92362 18.6887 2.08319 16.4185 2.08319H6.90643C3.64547 2.08319 1.00195 4.72669 1.00195 7.98763C1.00195 11.2486 3.47706 13.8921 7.42768 13.8921Z" fill="#39594D"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.03711 17.958C9.03711 16.3596 9.99942 14.9184 11.4758 14.3057L14.4713 13.0625C17.5013 11.805 20.8364 14.0316 20.8364 17.3123C20.8364 19.8539 18.7755 21.9141 16.2338 21.9134L12.9906 21.9126C10.807 21.912 9.03711 20.1417 9.03711 17.958Z" fill="#D18EE2"/>
+<path d="M4.40571 14.6705C2.5259 14.6705 1.00195 16.1943 1.00195 18.0741V18.515C1.00195 20.3947 2.52584 21.9186 4.40565 21.9186C6.28547 21.9186 7.80941 20.3947 7.80941 18.515V18.0741C7.80941 16.1943 6.28552 14.6705 4.40571 14.6705Z" fill="#FF7759"/>
+</g>
+</svg>

api/core/model_runtime/model_providers/cohere/_assets/icon_s_en.svg

@@ -0,0 +1,16 @@
+<svg width="22" height="22" viewBox="0 0 22 22" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Clip path group">
+<mask id="mask0_13224_9519" style="mask-type:luminance" maskUnits="userSpaceOnUse" x="0" y="0" width="22" height="22">
+<g id="clip0_2207_90691">
+<path id="Vector" d="M21.5 0.5H0.5V21.5H21.5V0.5Z" fill="white"/>
+</g>
+</mask>
+<g mask="url(#mask0_13224_9519)">
+<g id="Group">
+<path id="Vector_2" fill-rule="evenodd" clip-rule="evenodd" d="M7.30367 13.0035C7.8689 13.0035 8.99327 12.9725 10.5474 12.3326C12.3585 11.587 15.9617 10.2334 18.561 8.84305C20.3788 7.8706 21.1757 6.58448 21.1757 4.85248C21.1757 2.44869 19.2271 0.5 16.8233 0.5H6.75176C3.299 0.5 0.5 3.299 0.5 6.75176C0.5 10.2045 3.12069 13.0035 7.30367 13.0035Z" fill="#39594D"/>
+<path id="Vector_3" fill-rule="evenodd" clip-rule="evenodd" d="M9.00732 17.3086C9.00732 15.6162 10.0262 14.0902 11.5894 13.4414L14.7612 12.1251C17.9694 10.7936 21.5006 13.1513 21.5006 16.6249C21.5006 19.316 19.3185 21.4974 16.6273 21.4967L13.1933 21.4958C10.8813 21.4952 9.00732 19.6207 9.00732 17.3086Z" fill="#D18EE2"/>
+<path id="Vector_4" d="M4.10396 13.8277C2.11358 13.8277 0.5 15.4411 0.5 17.4315V17.8984C0.5 19.8887 2.11352 21.5022 4.1039 21.5022C6.09428 21.5022 7.70785 19.8887 7.70785 17.8984V17.4315C7.70785 15.4411 6.09434 13.8277 4.10396 13.8277Z" fill="#FF7759"/>
+</g>
+</g>
+</g>
+</svg>

api/core/model_runtime/model_providers/cohere/cohere.py

@@ -0,0 +1,31 @@
+import logging
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class CohereProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.RERANK)
+
+            # Use `rerank-english-v2.0` model for validate,
+            model_instance.validate_credentials(
+                model='rerank-english-v2.0',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/cohere/cohere.yaml

@@ -0,0 +1,31 @@
+provider: cohere
+label:
+  zh_Hans: Cohere
+  en_US: Cohere
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#ECE9E3"
+help:
+  title:
+    en_US: Get your API key from cohere
+    zh_Hans: 从 cohere 获取 API Key
+  url:
+    en_US: https://dashboard.cohere.com/api-keys
+supported_model_types:
+- rerank
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: api_key
+    label:
+      zh_Hans: API Key
+      en_US: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 请填写 API Key
+      en_US: Please fill in API Key
+    show_on: []

api/core/model_runtime/model_providers/cohere/rerank/rerank-english-v2.0.yaml

@@ -0,0 +1,4 @@
+model: rerank-english-v2.0
+model_type: rerank
+model_properties:
+  context_size: 5120

api/core/model_runtime/model_providers/cohere/rerank/rerank-multilingual-v2.0.yaml

@@ -0,0 +1,4 @@
+model: rerank-multilingual-v2.0
+model_type: rerank
+model_properties:
+  context_size: 5120

api/core/model_runtime/model_providers/cohere/rerank/rerank.py

@@ -0,0 +1,114 @@
+from typing import Optional
+
+import cohere
+
+from core.model_runtime.entities.rerank_entities import RerankResult, RerankDocument
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.rerank_model import RerankModel
+
+
+class CohereRerankModel(RerankModel):
+    """
+    Model class for Cohere rerank model.
+    """
+
+    def _invoke(self, model: str, credentials: dict,
+                query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None,
+                user: Optional[str] = None) \
+            -> RerankResult:
+        """
+        Invoke rerank model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param query: search query
+        :param docs: docs for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n
+        :param user: unique user id
+        :return: rerank result
+        """
+        if len(docs) == 0:
+            return RerankResult(
+                model=model,
+                docs=docs
+            )
+
+        # initialize client
+        client = cohere.Client(credentials.get('api_key'))
+        results = client.rerank(
+            query=query,
+            documents=docs,
+            model=model,
+            top_n=top_n
+        )
+
+        rerank_documents = []
+        for idx, result in enumerate(results):
+            # format document
+            rerank_document = RerankDocument(
+                index=result.index,
+                text=result.document['text'],
+                score=result.relevance_score,
+            )
+
+            # score threshold check
+            if score_threshold is not None:
+                if result.relevance_score >= score_threshold:
+                    rerank_documents.append(rerank_document)
+            else:
+                rerank_documents.append(rerank_document)
+
+        return RerankResult(
+            model=model,
+            docs=rerank_documents
+        )
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self.invoke(
+                model=model,
+                credentials=credentials,
+                query="What is the capital of the United States?",
+                docs=[
+                    "Carson City is the capital city of the American state of Nevada. At the 2010 United States "
+                    "Census, Carson City had a population of 55,274.",
+                    "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that "
+                    "are a political division controlled by the United States. Its capital is Saipan.",
+                ],
+                score_threshold=0.8
+            )
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the error type thrown to the caller
+        The value is the error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke error mapping
+        """
+        return {
+            InvokeConnectionError: [
+                cohere.CohereConnectionError,
+            ],
+            InvokeServerUnavailableError: [],
+            InvokeRateLimitError: [],
+            InvokeAuthorizationError: [],
+            InvokeBadRequestError: [
+                cohere.CohereAPIError,
+                cohere.CohereError,
+            ]
+        }

api/core/model_runtime/model_providers/google/_assets/icon_l_en.svg

@@ -0,0 +1,15 @@
+<svg width="68" height="24" viewBox="0 0 68 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g id="Gemini">
+<path id="Union" fill-rule="evenodd" clip-rule="evenodd" d="M50.6875 4.37014C48.3498 4.59292 46.5349 6.41319 46.3337 8.72764C46.1446 6.44662 44.2677 4.56074 41.9805 4.3737C44.2762 4.1997 46.152 2.28299 46.3373 0C46.4882 2.28911 48.405 4.20047 50.6875 4.37014ZM15.4567 9.41141L13.9579 10.9076C9.92941 6.64892 2.69298 9.97287 3.17317 15.8112C3.22394 23.108 14.5012 24.4317 15.3628 16.8809H9.52096L9.50061 14.9149H17.3595C18.8163 23.1364 8.44367 27.0292 3.19453 21.238C0.847044 18.7556 0.363651 14.7682 1.83717 11.7212C4.1129 6.62089 11.6505 5.29845 15.4567 9.41141ZM45.5915 23.5989H47.6945C47.6944 22.9155 47.6945 22.2307 47.6946 21.5452V21.5325C47.6948 19.8907 47.695 18.2453 47.6924 16.6072C47.6914 15.9407 47.6161 15.2823 47.4024 14.647C46.4188 11.2828 41.4255 11.4067 39.8332 14.214C38.5637 11.4171 34.4009 11.5236 32.8538 14.0084L32.8082 13.9976V12.4806L32.4233 12.4804H32.4224C31.8687 12.4801 31.3324 12.4798 30.7949 12.4811V23.5848L32.8977 23.5672C32.8981 22.9411 32.8979 22.3122 32.8977 21.6822V21.6812V21.6802V21.6791V21.6781V21.6771V21.676V21.676V21.6759V21.6758V21.6757V21.6757V21.6756C32.8973 20.204 32.8969 18.7261 32.904 17.2614C32.8889 15.3646 34.5674 13.5687 36.5358 14.124C37.7794 14.3298 38.1851 15.6148 38.1761 16.7257C38.1821 17.7019 38.18 18.6824 38.178 19.6633V19.6638C38.1752 20.9756 38.1724 22.2881 38.1891 23.5919L40.2846 23.5731C40.2929 22.7511 40.2881 21.9245 40.2832 21.0966C40.2753 19.7402 40.2674 18.3805 40.317 17.0328C40.4418 15.2122 42.0141 13.6186 43.9064 14.1168C45.2685 14.3231 45.6136 15.7748 45.5882 16.9545C45.5938 18.4959 45.5929 20.0492 45.5921 21.5968V21.5991V21.6014V21.6037V21.606V21.6083V21.6106C45.5917 22.2749 45.5913 22.9382 45.5915 23.5989ZM20.6167 18.4408C20.5625 21.9486 25.2121 23.6996 27.2993 20.0558L29.1566 20.9592C27.8157 23.7067 24.2337 24.7424 21.5381 23.4213C18.0052 21.7253 17.41 16.5007 20.0334 13.7517C21.4609 12.1752 23.7291 11.7901 25.7206 12.3653C28.3408 13.1257 29.4974 15.8937 29.326 18.4399C27.5547 18.4415 25.7971 18.4412 24.0364 18.4409C22.8993 18.4407 21.7609 18.4405 20.6167 18.4408ZM27.1041 16.6957C26.7048 13.1033 21.2867 13.2256 20.7494 16.6957H27.1041ZM53.543 23.5999H55.6206L55.6206 22.4361C55.6205 20.7877 55.6205 19.1443 55.6207 17.4939C55.6208 16.8853 55.7234 16.297 56.0063 15.7531C56.6115 14.3862 58.1745 13.7002 59.5927 14.1774C60.7512 14.4455 61.2852 15.6069 61.2762 16.7154C61.2774 18.3497 61.2771 19.9826 61.2769 21.6162V21.6166V21.617V21.6174V21.6179L61.2766 23.6007H63.3698C63.3913 22.0924 63.3869 20.584 63.3826 19.0755V19.0754V19.0753V19.0753V19.0752C63.3799 18.1682 63.3773 17.2612 63.3803 16.3541C63.3796 15.8622 63.3103 15.3765 63.1698 14.9052C62.3248 11.5142 57.3558 11.2385 55.5828 14.0038L55.5336 13.9905V12.4917H53.539C53.4898 12.7313 53.4934 23.4113 53.543 23.5999ZM49.6211 12.4944H51.7065V23.5994H49.6211V12.4944ZM65.1035 23.5991H67.1831C67.2367 23.2198 67.2133 12.6566 67.1634 12.4983H65.1035V23.5991ZM52.1504 8.67829C52.1709 10.4847 49.2418 10.7058 49.1816 8.65714C49.2189 6.5948 52.2437 6.81331 52.1504 8.67829ZM66.1387 10.1324C64.2712 10.1609 64.1316 7.19881 66.1559 7.17114C68.1709 7.19817 68.0215 10.2087 66.1387 10.1324Z" fill="url(#paint0_linear_14286_118464)"/>
+</g>
+<defs>
+<linearGradient id="paint0_linear_14286_118464" x1="-2" y1="0.999998" x2="67.9999" y2="27.5002" gradientUnits="userSpaceOnUse">
+<stop stop-color="#7798E0"/>
+<stop offset="0.210002" stop-color="#086FFF"/>
+<stop offset="0.345945" stop-color="#086FFF"/>
+<stop offset="0.591777" stop-color="#479AFF"/>
+<stop offset="0.895892" stop-color="#B7C4FA"/>
+<stop offset="1" stop-color="#B5C5F9"/>
+</linearGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/google/_assets/icon_s_en.svg

@@ -0,0 +1,11 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect width="24" height="24" rx="6" fill="url(#paint0_linear_7301_16076)"/>
+<path d="M20 12.0116C15.7043 12.42 12.3692 15.757 11.9995 20C11.652 15.8183 8.20301 12.361 4 12.0181C8.21855 11.6991 11.6656 8.1853 12.006 4C12.2833 8.19653 15.8057 11.7005 20 12.0116Z" fill="white" fill-opacity="0.88"/>
+<defs>
+<linearGradient id="paint0_linear_7301_16076" x1="-9" y1="29.5" x2="19.4387" y2="1.43791" gradientUnits="userSpaceOnUse">
+<stop offset="0.192878" stop-color="#1C7DFF"/>
+<stop offset="0.520213" stop-color="#1C69FF"/>
+<stop offset="1" stop-color="#F0DCD6"/>
+</linearGradient>
+</defs>
+</svg>

api/core/model_runtime/model_providers/google/google.py

@@ -0,0 +1,31 @@
+import logging
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class GoogleProvider(ModelProvider):
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.LLM)
+
+            # Use `gemini-pro` model for validate,
+            model_instance.validate_credentials(
+                model='gemini-pro',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/google/google.yaml

@@ -0,0 +1,32 @@
+provider: google
+label:
+  en_US: Google
+description:
+  en_US: Google's Gemini model.
+  zh_Hans: 谷歌提供的 Gemini 模型.
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#FCFDFF"
+help:
+  title:
+    en_US: Get your API Key from Google
+    zh_Hans: 从 Google 获取 API Key
+  url:
+    en_US: https://ai.google.dev/
+supported_model_types:
+- llm
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: google_api_key
+    label:
+      en_US: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API Key
+      en_US: Enter your API Key
+  

api/core/model_runtime/model_providers/google/llm/gemini-pro-vision.yaml

@@ -0,0 +1,34 @@
+model: gemini-pro-vision
+label:
+  en_US: Gemini Pro Vision
+model_type: llm
+features:
+- vision
+model_properties:
+  mode: chat
+  context_size: 12288
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens_to_sample
+  use_template: max_tokens
+  required: true
+  default: 4096
+  min: 1
+  max: 4096
+pricing:
+  input: '0.00'
+  output: '0.00'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/google/llm/gemini-pro.yaml

@@ -0,0 +1,34 @@
+model: gemini-pro
+label:
+  en_US: Gemini Pro
+model_type: llm
+features:
+- agent-thought
+model_properties:
+  mode: chat
+  context_size: 30720
+parameter_rules:
+- name: temperature
+  use_template: temperature
+- name: top_p
+  use_template: top_p
+- name: top_k
+  label:
+    zh_Hans: 取样数量
+    en_US: Top k
+  type: int
+  help:
+    zh_Hans: 仅从每个后续标记的前 K 个选项中采样。
+    en_US: Only sample from the top K options for each subsequent token.
+  required: false
+- name: max_tokens_to_sample
+  use_template: max_tokens
+  required: true
+  default: 2048
+  min: 1
+  max: 2048
+pricing:
+  input: '0.00'
+  output: '0.00'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/google/llm/llm.py

@@ -0,0 +1,342 @@
+from typing import Optional, Generator, Union, List
+
+import google.generativeai as genai
+import google.api_core.exceptions as exceptions
+import google.generativeai.client as client
+
+from google.generativeai.types import GenerateContentResponse, ContentType
+from google.generativeai.types.content_types import to_part
+
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, UserPromptMessage, AssistantPromptMessage, \
+    SystemPromptMessage, PromptMessageRole, PromptMessageContentType
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, \
+    LLMResultChunkDelta
+from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \
+    InvokeAuthorizationError, InvokeBadRequestError, InvokeError
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers import google
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+
+import logging
+logger = logging.getLogger(__name__)
+
+class GoogleLargeLanguageModel(LargeLanguageModel):
+
+    def _invoke(self, model: str, credentials: dict,
+                prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None,
+                stream: bool = True, user: Optional[str] = None) \
+            -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param tools: tools for tool calling
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        # invoke model
+        return self._generate(model, credentials, prompt_messages, model_parameters, stop, stream, user)
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: Optional[list[PromptMessageTool]] = None) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param prompt_messages: prompt messages
+        :param tools: tools for tool calling
+        :return:md = genai.GenerativeModel(model)
+        """
+        prompt = self._convert_messages_to_prompt(prompt_messages)
+
+        return self._get_num_tokens_by_gpt2(prompt)
+    
+    def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str:
+        """
+        Format a list of messages into a full prompt for the Google model
+
+        :param messages: List of PromptMessage to combine.
+        :return: Combined string with necessary human_prompt and ai_prompt tags.
+        """
+        messages = messages.copy()  # don't mutate the original list
+
+        text = "".join(
+            self._convert_one_message_to_text(message)
+            for message in messages
+        )
+
+        return text.rstrip()
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        
+        try:
+            ping_message = PromptMessage(content="ping", role="system")
+            self._generate(model, credentials, [ping_message], {"max_tokens_to_sample": 5})
+            
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+            
+
+    def _generate(self, model: str, credentials: dict,
+                  prompt_messages: list[PromptMessage], model_parameters: dict,
+                  stop: Optional[List[str]] = None, stream: bool = True,
+                  user: Optional[str] = None) -> Union[LLMResult, Generator]:
+        """
+        Invoke large language model
+
+        :param model: model name
+        :param credentials: credentials kwargs
+        :param prompt_messages: prompt messages
+        :param model_parameters: model parameters
+        :param stop: stop words
+        :param stream: is stream response
+        :param user: unique user id
+        :return: full response or stream response chunk generator result
+        """
+        config_kwargs = model_parameters.copy()
+        config_kwargs['max_output_tokens'] = config_kwargs.pop('max_tokens_to_sample', None)
+
+        if stop:
+            config_kwargs["stop_sequences"] = stop
+
+        google_model = genai.GenerativeModel(
+            model_name=model
+        )
+
+        history = []
+
+        # hack for gemini-pro-vision, which currently does not support multi-turn chat
+        if model == "gemini-pro-vision":
+            last_msg = prompt_messages[-1]
+            content = self._format_message_to_glm_content(last_msg)
+            history.append(content)
+        else:    
+            for msg in prompt_messages:     # makes message roles strictly alternating
+                content = self._format_message_to_glm_content(msg)
+                if history and history[-1]["role"] == content["role"]:
+                    history[-1]["parts"].extend(content["parts"])
+                else:
+                    history.append(content)
+
+
+        # Create a new ClientManager with tenant's API key
+        new_client_manager = client._ClientManager()
+        new_client_manager.configure(api_key=credentials["google_api_key"])
+        new_custom_client = new_client_manager.make_client("generative")
+
+        google_model._client = new_custom_client
+        
+        response = google_model.generate_content(
+            contents=history,
+            generation_config=genai.types.GenerationConfig(
+                **config_kwargs
+            ),
+            stream=stream
+        )
+
+        if stream:
+            return self._handle_generate_stream_response(model, credentials, response, prompt_messages)
+
+        return self._handle_generate_response(model, credentials, response, prompt_messages)
+
+    def _handle_generate_response(self, model: str, credentials: dict, response: GenerateContentResponse,
+                                  prompt_messages: list[PromptMessage]) -> LLMResult:
+        """
+        Handle llm response
+
+        :param model: model name
+        :param credentials: credentials
+        :param response: response
+        :param prompt_messages: prompt messages
+        :return: llm response
+        """
+        # transform assistant message to prompt message
+        assistant_prompt_message = AssistantPromptMessage(
+            content=response.text
+        )
+
+
+        # calculate num tokens
+        prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+        completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+        # transform usage
+        usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+        # transform response
+        result = LLMResult(
+            model=model,
+            prompt_messages=prompt_messages,
+            message=assistant_prompt_message,
+            usage=usage,
+        )
+
+        return result
+
+    def _handle_generate_stream_response(self, model: str, credentials: dict, response: GenerateContentResponse,
+                                         prompt_messages: list[PromptMessage]) -> Generator:
+        """
+        Handle llm stream response
+
+        :param model: model name
+        :param credentials: credentials
+        :param response: response
+        :param prompt_messages: prompt messages
+        :return: llm response chunk generator result
+        """
+        index = -1
+        for chunk in response:
+            content = chunk.text
+            index += 1
+
+            assistant_prompt_message = AssistantPromptMessage(
+                content=content if content else '',
+            )
+
+            if not response._done:
+                
+                # transform assistant message to prompt message
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=index,
+                        message=assistant_prompt_message
+                    )
+                )
+            else:
+                
+                # calculate num tokens
+                prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+                completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+                # transform usage
+                usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+                
+                yield LLMResultChunk(
+                    model=model,
+                    prompt_messages=prompt_messages,
+                    delta=LLMResultChunkDelta(
+                        index=index,
+                        message=assistant_prompt_message,
+                        finish_reason=chunk.candidates[0].finish_reason,
+                        usage=usage
+                    )
+                )
+
+    def _convert_one_message_to_text(self, message: PromptMessage) -> str:
+        """
+        Convert a single message to a string.
+
+        :param message: PromptMessage to convert.
+        :return: String representation of the message.
+        """
+        human_prompt = "\n\nuser:"
+        ai_prompt = "\n\nmodel:"
+
+        content = message.content
+        if isinstance(content, list):
+            content = "".join(
+                c.data for c in content if c.type != PromptMessageContentType.IMAGE
+            )
+
+        if isinstance(message, UserPromptMessage):
+            message_text = f"{human_prompt} {content}"
+        elif isinstance(message, AssistantPromptMessage):
+            message_text = f"{ai_prompt} {content}"
+        elif isinstance(message, SystemPromptMessage):
+            message_text = f"{human_prompt} {content}"
+        else:
+            raise ValueError(f"Got unknown type {message}")
+
+        return message_text
+
+    def _format_message_to_glm_content(self, message: PromptMessage) -> ContentType:
+        """
+        Format a single message into glm.Content for Google API
+
+        :param message: one PromptMessage
+        :return: glm Content representation of message
+        """
+
+        parts = []
+        if (isinstance(message.content, str)):
+            parts.append(to_part(message.content))
+        else:
+            for c in message.content:
+                if c.type == PromptMessageContentType.TEXT:
+                    parts.append(to_part(c.data))
+                else:
+                    metadata, data = c.data.split(',', 1)
+                    mime_type = metadata.split(';', 1)[0].split(':')[1]
+                    blob = {"inline_data":{"mime_type":mime_type,"data":data}}
+                    parts.append(blob)
+
+        glm_content = {
+            "role": "user" if message.role in (PromptMessageRole.USER, PromptMessageRole.SYSTEM) else "model",
+            "parts": parts
+        }
+        
+        return glm_content
+    
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        """
+        Map model invoke error to unified error
+        The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller
+        The value is the md = genai.GenerativeModel(model)error type thrown by the model,
+        which needs to be converted into a unified error type for the caller.
+
+        :return: Invoke emd = genai.GenerativeModel(model)rror mapping
+        """
+        return {
+            InvokeConnectionError: [
+                exceptions.RetryError
+            ],
+            InvokeServerUnavailableError: [
+                exceptions.ServiceUnavailable,
+                exceptions.InternalServerError,
+                exceptions.BadGateway,
+                exceptions.GatewayTimeout,
+                exceptions.DeadlineExceeded
+            ],
+            InvokeRateLimitError: [
+                exceptions.ResourceExhausted,
+                exceptions.TooManyRequests
+            ],
+            InvokeAuthorizationError: [
+                exceptions.Unauthenticated,
+                exceptions.PermissionDenied,
+                exceptions.Unauthenticated,
+                exceptions.Forbidden
+            ],
+            InvokeBadRequestError: [
+                exceptions.BadRequest,
+                exceptions.InvalidArgument,
+                exceptions.FailedPrecondition,
+                exceptions.OutOfRange,
+                exceptions.NotFound,
+                exceptions.MethodNotAllowed,
+                exceptions.Conflict,
+                exceptions.AlreadyExists,
+                exceptions.Aborted,
+                exceptions.LengthRequired,
+                exceptions.PreconditionFailed,
+                exceptions.RequestRangeNotSatisfiable,
+                exceptions.Cancelled,
+            ]
+        }

api/core/model_runtime/model_providers/huggingface_hub/_assets/icon_l_en.svg

@@ -0,0 +1,42 @@
+<svg width="120" height="24" viewBox="0 0 120 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_8587_60377)">
+<g clip-path="url(#clip1_8587_60377)">
+<path d="M11.9286 20.2062C16.7767 20.2062 20.7069 16.2761 20.7069 11.428C20.7069 6.57993 16.7767 2.64978 11.9286 2.64978C7.08054 2.64978 3.15039 6.57993 3.15039 11.428C3.15039 16.2761 7.08054 20.2062 11.9286 20.2062Z" fill="#FFD21E"/>
+<path d="M20.7095 11.4326C20.7095 6.58451 16.7793 2.65436 11.9313 2.65436C7.08318 2.65436 3.15303 6.58451 3.15303 11.4326C3.15303 16.2807 7.08318 20.2108 11.9313 20.2108C16.7793 20.2108 20.7095 16.2807 20.7095 11.4326ZM2.14258 11.4326C2.14258 6.02647 6.52511 1.64392 11.9313 1.64392C17.3374 1.64392 21.7199 6.02647 21.7199 11.4326C21.7199 16.8387 17.3374 21.2213 11.9313 21.2213C6.52511 21.2213 2.14258 16.8387 2.14258 11.4326Z" fill="#FF9D0B"/>
+<path d="M14.7822 9.03703C15.1041 9.1507 15.2322 9.81254 15.5574 9.6396C16.1734 9.31212 16.4072 8.54734 16.0797 7.93142C15.7522 7.31553 14.9874 7.08172 14.3715 7.4092C13.7556 7.73669 13.5218 8.50147 13.8493 9.11738C14.0038 9.40809 14.4944 8.9354 14.7822 9.03703Z" fill="#3A3B45"/>
+<path d="M8.83422 9.03703C8.5123 9.1507 8.38422 9.81254 8.05901 9.6396C7.4431 9.31212 7.20928 8.54734 7.53676 7.93142C7.86425 7.31553 8.62903 7.08172 9.24494 7.4092C9.86086 7.73669 10.0947 8.50147 9.76719 9.11738C9.61262 9.40809 9.122 8.9354 8.83422 9.03703Z" fill="#3A3B45"/>
+<path d="M11.8679 15.1044C14.3507 15.1044 15.1519 12.8908 15.1519 11.7541C15.1519 11.1633 14.7547 11.3492 14.1187 11.6641C13.5309 11.9551 12.739 12.3563 11.8679 12.3563C10.0543 12.3563 8.58398 10.6173 8.58398 11.7541C8.58398 12.8908 9.38514 15.1044 11.8679 15.1044Z" fill="#3A3B45"/>
+<mask id="mask0_8587_60377" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="8" y="11" width="8" height="5">
+<path d="M11.8562 15.1005C14.339 15.1005 15.1402 12.8869 15.1402 11.7502C15.1402 11.1594 14.743 11.3453 14.1069 11.6602C13.5191 11.9512 12.7273 12.3524 11.8562 12.3524C10.0425 12.3524 8.57227 10.6134 8.57227 11.7502C8.57227 12.8869 9.37342 15.1005 11.8562 15.1005Z" fill="white"/>
+</mask>
+<g mask="url(#mask0_8587_60377)">
+<path d="M11.9194 17.6824C13.1294 17.6824 14.1103 16.7016 14.1103 15.4916C14.1103 14.5491 13.5152 13.7457 12.6803 13.4364C12.6496 13.425 12.6185 13.4143 12.5872 13.4043C12.3766 13.337 12.1523 14.0606 11.9194 14.0606C11.7018 14.0606 11.4917 13.3324 11.2933 13.3915C10.3884 13.6609 9.72852 14.4991 9.72852 15.4916C9.72852 16.7016 10.7094 17.6824 11.9194 17.6824Z" fill="#F94040"/>
+</g>
+<path d="M17.8698 10.2273C18.3232 10.2273 18.6908 9.85972 18.6908 9.40631C18.6908 8.9529 18.3232 8.58533 17.8698 8.58533C17.4164 8.58533 17.0488 8.9529 17.0488 9.40631C17.0488 9.85972 17.4164 10.2273 17.8698 10.2273Z" fill="#FF9D0B"/>
+<path d="M6.11981 10.2273C6.57323 10.2273 6.9408 9.85972 6.9408 9.40631C6.9408 8.9529 6.57323 8.58533 6.11981 8.58533C5.66638 8.58533 5.29883 8.9529 5.29883 9.40631C5.29883 9.85972 5.66638 10.2273 6.11981 10.2273Z" fill="#FF9D0B"/>
+<path d="M4.42915 13.0092C4.02018 13.0092 3.65465 13.1771 3.39976 13.4818C3.24214 13.6705 3.07743 13.9746 3.06404 14.4301C2.89252 14.3808 2.72757 14.3533 2.57347 14.3533C2.18193 14.3533 1.82827 14.5033 1.57819 14.7759C1.25687 15.1258 1.11414 15.5557 1.17628 15.9859C1.20584 16.1908 1.2743 16.3744 1.3766 16.5444C1.16087 16.719 1.00198 16.962 0.925188 17.2543C0.865067 17.4834 0.803429 17.9606 1.12526 18.4522C1.10479 18.4842 1.0856 18.5176 1.06766 18.5517C0.874161 18.919 0.861783 19.334 1.03255 19.7205C1.29147 20.3063 1.93487 20.7678 3.18429 21.2632C3.96157 21.5714 4.67267 21.7684 4.67899 21.7702C5.70661 22.0367 6.63596 22.1721 7.44053 22.1721C8.91931 22.1721 9.97801 21.7192 10.5873 20.8259C11.5679 19.3876 11.4277 18.072 10.1589 16.8039C9.45662 16.1021 8.98979 15.0674 8.89254 14.8403C8.69651 14.1679 8.17815 13.4204 7.3165 13.4204C7.244 13.4204 7.17049 13.4262 7.09824 13.4376C6.72084 13.4969 6.39093 13.7142 6.15525 14.0411C5.90087 13.7248 5.65381 13.4732 5.43025 13.3312C5.09327 13.1175 4.75654 13.0092 4.42915 13.0092ZM4.42915 14.0196C4.55799 14.0196 4.71536 14.0744 4.88891 14.1846C5.42773 14.5263 6.46747 16.3136 6.84816 17.0087C6.97573 17.2417 7.19373 17.3402 7.39001 17.3402C7.77953 17.3402 8.08368 16.9529 7.42563 16.4608C6.43615 15.7204 6.78324 14.5102 7.25562 14.4356C7.27633 14.4324 7.29679 14.4308 7.3165 14.4308C7.74594 14.4308 7.93539 15.171 7.93539 15.171C7.93539 15.171 8.49063 16.5654 9.44449 17.5185C10.3984 18.4719 10.4476 19.237 9.75243 20.2566C9.27828 20.9517 8.37064 21.1617 7.44053 21.1617C6.47581 21.1617 5.48684 20.9358 4.93261 20.7921C4.90533 20.785 1.53474 19.8329 1.96165 19.0226C2.03339 18.8864 2.15161 18.8318 2.3004 18.8318C2.90162 18.8318 3.99517 19.7266 4.46528 19.7266C4.57036 19.7266 4.64438 19.6819 4.67469 19.5727C4.87501 18.8541 1.62896 18.5519 1.90254 17.5109C1.95079 17.3268 2.08164 17.252 2.26554 17.2523C3.06 17.2523 4.84243 18.6495 5.21604 18.6495C5.24458 18.6495 5.26504 18.6411 5.27616 18.6234C5.46334 18.3213 5.36078 18.1104 4.0414 17.3119C2.72201 16.5131 1.79594 16.0327 2.32263 15.4592C2.38326 15.393 2.46915 15.3637 2.57347 15.3637C3.3745 15.364 5.26706 17.0863 5.26706 17.0863C5.26706 17.0863 5.77784 17.6175 6.08679 17.6175C6.15777 17.6175 6.21814 17.5895 6.25907 17.5203C6.47808 17.151 4.22479 15.4433 4.09773 14.7388C4.01159 14.2613 4.1581 14.0196 4.42915 14.0196Z" fill="#FF9D0B"/>
+<path d="M9.75883 20.2539C10.454 19.2344 10.4048 18.4692 9.4509 17.5159C8.49704 16.5628 7.9418 15.1684 7.9418 15.1684C7.9418 15.1684 7.73441 14.3585 7.26203 14.433C6.78964 14.5075 6.44281 15.7178 7.43228 16.4582C8.42176 17.1984 7.23525 17.7013 6.85456 17.0061C6.47388 16.3109 5.43438 14.5237 4.89531 14.1819C4.35649 13.8402 3.97707 14.0316 4.10414 14.7362C4.2312 15.4407 6.48474 17.1483 6.26547 17.5179C6.04621 17.8872 5.27347 17.0837 5.27347 17.0837C5.27347 17.0837 2.85548 14.8832 2.32903 15.4566C1.80258 16.03 2.72842 16.5105 4.0478 17.3093C5.36744 18.1078 5.46975 18.3187 5.28257 18.6208C5.09513 18.9229 2.18251 16.4673 1.90893 17.5083C1.63561 18.5493 4.88142 18.8514 4.6811 19.5701C4.48078 20.2891 2.3947 18.2098 1.96804 19.0199C1.54113 19.8303 4.91173 20.7824 4.93901 20.7895C6.02777 21.0719 8.79285 21.6703 9.75883 20.2539Z" fill="#FFD21E"/>
+<path d="M19.5568 13.0092C19.9658 13.0092 20.3313 13.1771 20.5862 13.4818C20.7439 13.6705 20.9086 13.9746 20.9219 14.4301C21.0935 14.3808 21.2584 14.3533 21.4125 14.3533C21.8041 14.3533 22.1577 14.5033 22.4078 14.7759C22.7291 15.1258 22.8718 15.5557 22.8097 15.9859C22.7802 16.1908 22.7117 16.3744 22.6094 16.5444C22.8251 16.719 22.984 16.962 23.0608 17.2543C23.1209 17.4834 23.1826 17.9606 22.8607 18.4522C22.8812 18.4842 22.9004 18.5176 22.9183 18.5517C23.1118 18.919 23.1242 19.334 22.9534 19.7205C22.6945 20.3063 22.0511 20.7678 20.8017 21.2632C20.0244 21.5714 19.3133 21.7684 19.307 21.7702C18.2794 22.0367 17.35 22.1721 16.5455 22.1721C15.0667 22.1721 14.008 21.7192 13.3987 20.8259C12.418 19.3876 12.5582 18.072 13.8271 16.8039C14.5294 16.1021 14.9962 15.0674 15.0935 14.8403C15.2895 14.1679 15.8078 13.4204 16.6695 13.4204C16.742 13.4204 16.8155 13.4262 16.8877 13.4376C17.2651 13.4969 17.5951 13.7142 17.8307 14.0411C18.0851 13.7248 18.3322 13.4732 18.5557 13.3312C18.8927 13.1175 19.2295 13.0092 19.5568 13.0092ZM19.5568 14.0196C19.428 14.0196 19.2706 14.0744 19.0971 14.1846C18.5583 14.5263 17.5185 16.3136 17.1378 17.0087C17.0103 17.2417 16.7923 17.3402 16.596 17.3402C16.2065 17.3402 15.9023 16.9529 16.5604 16.4608C17.5498 15.7204 17.2028 14.5102 16.7304 14.4356C16.7097 14.4324 16.6892 14.4308 16.6695 14.4308C16.2401 14.4308 16.0506 15.171 16.0506 15.171C16.0506 15.171 15.4954 16.5654 14.5415 17.5185C13.5876 18.4719 13.5384 19.237 14.2336 20.2566C14.7077 20.9517 15.6153 21.1617 16.5455 21.1617C17.5102 21.1617 18.4992 20.9358 19.0534 20.7921C19.0807 20.785 22.4513 19.8329 22.0243 19.0226C21.9526 18.8864 21.8344 18.8318 21.6856 18.8318C21.0844 18.8318 19.9908 19.7266 19.5207 19.7266C19.4156 19.7266 19.3416 19.6819 19.3113 19.5727C19.111 18.8541 22.357 18.5519 22.0835 17.5109C22.0352 17.3268 21.9043 17.252 21.7204 17.2523C20.926 17.2523 19.1436 18.6495 18.77 18.6495C18.7414 18.6495 18.7209 18.6411 18.7098 18.6234C18.5226 18.3213 18.6252 18.1104 19.9446 17.3119C21.264 16.5131 22.1901 16.0327 21.6634 15.4592C21.6027 15.393 21.5168 15.3637 21.4125 15.3637C20.6115 15.364 18.7189 17.0863 18.7189 17.0863C18.7189 17.0863 18.2081 17.6175 17.8992 17.6175C17.8282 17.6175 17.7678 17.5895 17.7269 17.5203C17.5079 17.151 19.7612 15.4433 19.8883 14.7388C19.9744 14.2613 19.8279 14.0196 19.5568 14.0196Z" fill="#FF9D0B"/>
+<path d="M14.2354 20.2539C13.5402 19.2344 13.5895 18.4692 14.5433 17.5159C15.4972 16.5628 16.0524 15.1684 16.0524 15.1684C16.0524 15.1684 16.2598 14.3585 16.7322 14.433C17.2046 14.5075 17.5514 15.7178 16.5619 16.4582C15.5724 17.1984 16.759 17.7013 17.1396 17.0061C17.5203 16.3109 18.5598 14.5237 19.0989 14.1819C19.6377 13.8402 20.0171 14.0316 19.8901 14.7362C19.763 15.4407 17.5095 17.1483 17.7287 17.5179C17.948 17.8872 18.7207 17.0837 18.7207 17.0837C18.7207 17.0837 21.1387 14.8832 21.6652 15.4566C22.1916 16.03 21.2658 16.5105 19.9464 17.3093C18.6268 18.1078 18.5245 18.3187 18.7116 18.6208C18.8991 18.9229 21.8117 16.4673 22.0853 17.5083C22.3586 18.5493 19.1128 18.8514 19.3131 19.5701C19.5134 20.2891 21.5995 18.2098 22.0262 19.0199C22.4531 19.8303 19.0825 20.7824 19.0552 20.7895C17.9664 21.0719 15.2014 21.6703 14.2354 20.2539Z" fill="#FFD21E"/>
+</g>
+<path d="M33.1528 17V7.22003H35.3578V10.985H38.7328V7.22003H40.9528V17H38.7328V12.92H35.3578V17H33.1528Z" fill="#1D2939"/>
+<path d="M45.3153 17.18C44.5053 17.18 43.9153 16.915 43.5453 16.385C43.1853 15.845 43.0053 15.11 43.0053 14.18V9.56003H45.2103V13.895C45.2103 14.425 45.2853 14.795 45.4353 15.005C45.5853 15.205 45.8203 15.305 46.1403 15.305C46.4203 15.305 46.6553 15.24 46.8453 15.11C47.0353 14.98 47.2403 14.77 47.4603 14.48V9.56003H49.6653V17H47.8653L47.7003 15.965H47.6553C47.3453 16.335 47.0053 16.63 46.6353 16.85C46.2653 17.07 45.8253 17.18 45.3153 17.18Z" fill="#1D2939"/>
+<path d="M54.2606 20.165C53.6806 20.165 53.1556 20.1 52.6856 19.97C52.2156 19.84 51.8406 19.635 51.5606 19.355C51.2806 19.075 51.1406 18.715 51.1406 18.275C51.1406 17.675 51.4956 17.175 52.2056 16.775V16.715C52.0156 16.585 51.8506 16.42 51.7106 16.22C51.5806 16.02 51.5156 15.765 51.5156 15.455C51.5156 15.185 51.5956 14.925 51.7556 14.675C51.9156 14.425 52.1156 14.22 52.3556 14.06V14C52.0956 13.82 51.8606 13.56 51.6506 13.22C51.4506 12.88 51.3506 12.495 51.3506 12.065C51.3506 11.465 51.4956 10.97 51.7856 10.58C52.0756 10.18 52.4556 9.88003 52.9256 9.68003C53.3956 9.48003 53.8956 9.38003 54.4256 9.38003C54.8656 9.38003 55.2506 9.44003 55.5806 9.56003H58.2956V11.165H57.1106C57.1806 11.275 57.2356 11.415 57.2756 11.585C57.3256 11.755 57.3506 11.94 57.3506 12.14C57.3506 12.71 57.2206 13.18 56.9606 13.55C56.7006 13.92 56.3506 14.195 55.9106 14.375C55.4706 14.555 54.9756 14.645 54.4256 14.645C54.1356 14.645 53.8356 14.595 53.5256 14.495C53.3456 14.645 53.2556 14.83 53.2556 15.05C53.2556 15.24 53.3406 15.38 53.5106 15.47C53.6806 15.56 53.9706 15.605 54.3806 15.605H55.5806C56.5006 15.605 57.2006 15.755 57.6806 16.055C58.1706 16.345 58.4156 16.825 58.4156 17.495C58.4156 18.005 58.2456 18.46 57.9056 18.86C57.5656 19.27 57.0856 19.59 56.4656 19.82C55.8456 20.05 55.1106 20.165 54.2606 20.165ZM54.4256 13.31C54.7156 13.31 54.9556 13.205 55.1456 12.995C55.3456 12.785 55.4456 12.475 55.4456 12.065C55.4456 11.675 55.3456 11.38 55.1456 11.18C54.9556 10.97 54.7156 10.865 54.4256 10.865C54.1356 10.865 53.8906 10.965 53.6906 11.165C53.5006 11.365 53.4056 11.665 53.4056 12.065C53.4056 12.475 53.5006 12.785 53.6906 12.995C53.8906 13.205 54.1356 13.31 54.4256 13.31ZM54.6056 18.785C55.1056 18.785 55.5156 18.695 55.8356 18.515C56.1556 18.335 56.3156 18.12 56.3156 17.87C56.3156 17.64 56.2156 17.485 56.0156 17.405C55.8256 17.325 55.5456 17.285 55.1756 17.285H54.4106C54.1606 17.285 53.9506 17.275 53.7806 17.255C53.6206 17.245 53.4806 17.225 53.3606 17.195C53.0906 17.435 52.9556 17.68 52.9556 17.93C52.9556 18.21 53.1056 18.42 53.4056 18.56C53.7156 18.71 54.1156 18.785 54.6056 18.785Z" fill="#1D2939"/>
+<path d="M62.2733 20.165C61.6933 20.165 61.1683 20.1 60.6983 19.97C60.2283 19.84 59.8533 19.635 59.5733 19.355C59.2933 19.075 59.1533 18.715 59.1533 18.275C59.1533 17.675 59.5083 17.175 60.2183 16.775V16.715C60.0283 16.585 59.8633 16.42 59.7233 16.22C59.5933 16.02 59.5283 15.765 59.5283 15.455C59.5283 15.185 59.6083 14.925 59.7683 14.675C59.9283 14.425 60.1283 14.22 60.3683 14.06V14C60.1083 13.82 59.8733 13.56 59.6633 13.22C59.4633 12.88 59.3633 12.495 59.3633 12.065C59.3633 11.465 59.5083 10.97 59.7983 10.58C60.0883 10.18 60.4683 9.88003 60.9383 9.68003C61.4083 9.48003 61.9083 9.38003 62.4383 9.38003C62.8783 9.38003 63.2633 9.44003 63.5933 9.56003H66.3083V11.165H65.1233C65.1933 11.275 65.2483 11.415 65.2883 11.585C65.3383 11.755 65.3633 11.94 65.3633 12.14C65.3633 12.71 65.2333 13.18 64.9733 13.55C64.7133 13.92 64.3633 14.195 63.9233 14.375C63.4833 14.555 62.9883 14.645 62.4383 14.645C62.1483 14.645 61.8483 14.595 61.5383 14.495C61.3583 14.645 61.2683 14.83 61.2683 15.05C61.2683 15.24 61.3533 15.38 61.5233 15.47C61.6933 15.56 61.9833 15.605 62.3933 15.605H63.5933C64.5133 15.605 65.2133 15.755 65.6933 16.055C66.1833 16.345 66.4283 16.825 66.4283 17.495C66.4283 18.005 66.2583 18.46 65.9183 18.86C65.5783 19.27 65.0983 19.59 64.4783 19.82C63.8583 20.05 63.1233 20.165 62.2733 20.165ZM62.4383 13.31C62.7283 13.31 62.9683 13.205 63.1583 12.995C63.3583 12.785 63.4583 12.475 63.4583 12.065C63.4583 11.675 63.3583 11.38 63.1583 11.18C62.9683 10.97 62.7283 10.865 62.4383 10.865C62.1483 10.865 61.9033 10.965 61.7033 11.165C61.5133 11.365 61.4183 11.665 61.4183 12.065C61.4183 12.475 61.5133 12.785 61.7033 12.995C61.9033 13.205 62.1483 13.31 62.4383 13.31ZM62.6183 18.785C63.1183 18.785 63.5283 18.695 63.8483 18.515C64.1683 18.335 64.3283 18.12 64.3283 17.87C64.3283 17.64 64.2283 17.485 64.0283 17.405C63.8383 17.325 63.5583 17.285 63.1883 17.285H62.4233C62.1733 17.285 61.9633 17.275 61.7933 17.255C61.6333 17.245 61.4933 17.225 61.3733 17.195C61.1033 17.435 60.9683 17.68 60.9683 17.93C60.9683 18.21 61.1183 18.42 61.4183 18.56C61.7283 18.71 62.1283 18.785 62.6183 18.785Z" fill="#1D2939"/>
+<path d="M67.631 17V9.56003H69.836V17H67.631ZM68.726 8.46503C68.356 8.46503 68.056 8.36003 67.826 8.15003C67.596 7.94003 67.481 7.66003 67.481 7.31003C67.481 6.96003 67.596 6.68003 67.826 6.47003C68.056 6.26003 68.356 6.15503 68.726 6.15503C69.096 6.15503 69.396 6.26003 69.626 6.47003C69.856 6.68003 69.971 6.96003 69.971 7.31003C69.971 7.66003 69.856 7.94003 69.626 8.15003C69.396 8.36003 69.096 8.46503 68.726 8.46503Z" fill="#1D2939"/>
+<path d="M71.7765 17V9.56003H73.5765L73.7265 10.505H73.7865C74.1065 10.205 74.4565 9.94503 74.8365 9.72503C75.2265 9.49503 75.6715 9.38003 76.1715 9.38003C76.9815 9.38003 77.5665 9.65003 77.9265 10.19C78.2965 10.72 78.4815 11.45 78.4815 12.38V17H76.2765V12.665C76.2765 12.125 76.2015 11.755 76.0515 11.555C75.9115 11.355 75.6815 11.255 75.3615 11.255C75.0815 11.255 74.8415 11.32 74.6415 11.45C74.4415 11.57 74.2215 11.745 73.9815 11.975V17H71.7765Z" fill="#1D2939"/>
+<path d="M83.0155 20.165C82.4355 20.165 81.9105 20.1 81.4405 19.97C80.9705 19.84 80.5955 19.635 80.3155 19.355C80.0355 19.075 79.8955 18.715 79.8955 18.275C79.8955 17.675 80.2505 17.175 80.9605 16.775V16.715C80.7705 16.585 80.6055 16.42 80.4655 16.22C80.3355 16.02 80.2705 15.765 80.2705 15.455C80.2705 15.185 80.3505 14.925 80.5105 14.675C80.6705 14.425 80.8705 14.22 81.1105 14.06V14C80.8505 13.82 80.6155 13.56 80.4055 13.22C80.2055 12.88 80.1055 12.495 80.1055 12.065C80.1055 11.465 80.2505 10.97 80.5405 10.58C80.8305 10.18 81.2105 9.88003 81.6805 9.68003C82.1505 9.48003 82.6505 9.38003 83.1805 9.38003C83.6205 9.38003 84.0055 9.44003 84.3355 9.56003H87.0505V11.165H85.8655C85.9355 11.275 85.9905 11.415 86.0305 11.585C86.0805 11.755 86.1055 11.94 86.1055 12.14C86.1055 12.71 85.9755 13.18 85.7155 13.55C85.4555 13.92 85.1055 14.195 84.6655 14.375C84.2255 14.555 83.7305 14.645 83.1805 14.645C82.8905 14.645 82.5905 14.595 82.2805 14.495C82.1005 14.645 82.0105 14.83 82.0105 15.05C82.0105 15.24 82.0955 15.38 82.2655 15.47C82.4355 15.56 82.7255 15.605 83.1355 15.605H84.3355C85.2555 15.605 85.9555 15.755 86.4355 16.055C86.9255 16.345 87.1705 16.825 87.1705 17.495C87.1705 18.005 87.0005 18.46 86.6605 18.86C86.3205 19.27 85.8405 19.59 85.2205 19.82C84.6005 20.05 83.8655 20.165 83.0155 20.165ZM83.1805 13.31C83.4705 13.31 83.7105 13.205 83.9005 12.995C84.1005 12.785 84.2005 12.475 84.2005 12.065C84.2005 11.675 84.1005 11.38 83.9005 11.18C83.7105 10.97 83.4705 10.865 83.1805 10.865C82.8905 10.865 82.6455 10.965 82.4455 11.165C82.2555 11.365 82.1605 11.665 82.1605 12.065C82.1605 12.475 82.2555 12.785 82.4455 12.995C82.6455 13.205 82.8905 13.31 83.1805 13.31ZM83.3605 18.785C83.8605 18.785 84.2705 18.695 84.5905 18.515C84.9105 18.335 85.0705 18.12 85.0705 17.87C85.0705 17.64 84.9705 17.485 84.7705 17.405C84.5805 17.325 84.3005 17.285 83.9305 17.285H83.1655C82.9155 17.285 82.7055 17.275 82.5355 17.255C82.3755 17.245 82.2355 17.225 82.1155 17.195C81.8455 17.435 81.7105 17.68 81.7105 17.93C81.7105 18.21 81.8605 18.42 82.1605 18.56C82.4705 18.71 82.8705 18.785 83.3605 18.785Z" fill="#1D2939"/>
+<path d="M91.5562 17V7.22003H97.7212V9.08003H93.7612V11.345H97.1512V13.205H93.7612V17H91.5562Z" fill="#1D2939"/>
+<path d="M100.546 17.18C99.8661 17.18 99.3261 16.965 98.9261 16.535C98.5261 16.095 98.3261 15.56 98.3261 14.93C98.3261 14.15 98.6561 13.54 99.3161 13.1C99.9761 12.66 101.041 12.365 102.511 12.215C102.491 11.885 102.391 11.625 102.211 11.435C102.041 11.235 101.751 11.135 101.341 11.135C101.031 11.135 100.716 11.195 100.396 11.315C100.076 11.435 99.7361 11.6 99.3761 11.81L98.5811 10.355C99.0511 10.065 99.5511 9.83003 100.081 9.65003C100.621 9.47003 101.181 9.38003 101.761 9.38003C102.711 9.38003 103.441 9.65503 103.951 10.205C104.461 10.755 104.716 11.6 104.716 12.74V17H102.916L102.766 16.235H102.706C102.396 16.515 102.061 16.745 101.701 16.925C101.351 17.095 100.966 17.18 100.546 17.18ZM101.296 15.47C101.546 15.47 101.761 15.415 101.941 15.305C102.131 15.185 102.321 15.03 102.511 14.84V13.535C101.731 13.635 101.191 13.795 100.891 14.015C100.591 14.225 100.441 14.475 100.441 14.765C100.441 15.005 100.516 15.185 100.666 15.305C100.826 15.415 101.036 15.47 101.296 15.47Z" fill="#1D2939"/>
+<path d="M109.821 17.18C109.131 17.18 108.506 17.03 107.946 16.73C107.396 16.42 106.956 15.975 106.626 15.395C106.306 14.805 106.146 14.1 106.146 13.28C106.146 12.45 106.326 11.745 106.686 11.165C107.046 10.585 107.521 10.145 108.111 9.84503C108.701 9.53503 109.336 9.38003 110.016 9.38003C110.476 9.38003 110.881 9.45503 111.231 9.60503C111.591 9.75503 111.911 9.94503 112.191 10.175L111.156 11.6C110.806 11.31 110.471 11.165 110.151 11.165C109.621 11.165 109.196 11.355 108.876 11.735C108.566 12.115 108.411 12.63 108.411 13.28C108.411 13.92 108.566 14.435 108.876 14.825C109.196 15.205 109.596 15.395 110.076 15.395C110.316 15.395 110.551 15.345 110.781 15.245C111.011 15.135 111.221 15.005 111.411 14.855L112.281 16.295C111.911 16.615 111.511 16.845 111.081 16.985C110.651 17.115 110.231 17.18 109.821 17.18Z" fill="#1D2939"/>
+<path d="M116.488 17.18C115.778 17.18 115.138 17.025 114.568 16.715C113.998 16.405 113.548 15.96 113.218 15.38C112.888 14.8 112.723 14.1 112.723 13.28C112.723 12.47 112.888 11.775 113.218 11.195C113.558 10.615 113.998 10.17 114.538 9.86003C115.078 9.54003 115.643 9.38003 116.233 9.38003C116.943 9.38003 117.528 9.54003 117.988 9.86003C118.458 10.17 118.808 10.595 119.038 11.135C119.278 11.665 119.398 12.27 119.398 12.95C119.398 13.14 119.388 13.33 119.368 13.52C119.348 13.7 119.328 13.835 119.308 13.925H114.853C114.953 14.465 115.178 14.865 115.528 15.125C115.878 15.375 116.298 15.5 116.788 15.5C117.318 15.5 117.853 15.335 118.393 15.005L119.128 16.34C118.748 16.6 118.323 16.805 117.853 16.955C117.383 17.105 116.928 17.18 116.488 17.18ZM114.838 12.47H117.523C117.523 12.06 117.423 11.725 117.223 11.465C117.033 11.195 116.718 11.06 116.278 11.06C115.938 11.06 115.633 11.18 115.363 11.42C115.093 11.65 114.918 12 114.838 12.47Z" fill="#1D2939"/>
+</g>
+<defs>
+<clipPath id="clip0_8587_60377">
+<rect width="119.998" height="24" rx="6" fill="white"/>
+</clipPath>
+<clipPath id="clip1_8587_60377">
+<rect width="23.998" height="22.2298" fill="white" transform="translate(0 0.885132)"/>
+</clipPath>
+</defs>
+</svg>

api/core/model_runtime/model_providers/huggingface_hub/_assets/icon_s_en.svg

@@ -0,0 +1,19 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M11.9286 20.2062C16.7767 20.2062 20.7069 16.2761 20.7069 11.428C20.7069 6.57993 16.7767 2.64978 11.9286 2.64978C7.08054 2.64978 3.15039 6.57993 3.15039 11.428C3.15039 16.2761 7.08054 20.2062 11.9286 20.2062Z" fill="#FFD21E"/>
+<path d="M20.7095 11.4326C20.7095 6.58451 16.7793 2.65436 11.9313 2.65436C7.08318 2.65436 3.15303 6.58451 3.15303 11.4326C3.15303 16.2807 7.08318 20.2108 11.9313 20.2108C16.7793 20.2108 20.7095 16.2807 20.7095 11.4326ZM2.14258 11.4326C2.14258 6.02647 6.52511 1.64392 11.9313 1.64392C17.3374 1.64392 21.7199 6.02647 21.7199 11.4326C21.7199 16.8387 17.3374 21.2213 11.9313 21.2213C6.52511 21.2213 2.14258 16.8387 2.14258 11.4326Z" fill="#FF9D0B"/>
+<path d="M14.7822 9.03703C15.1041 9.1507 15.2322 9.81254 15.5574 9.6396C16.1734 9.31212 16.4072 8.54734 16.0797 7.93142C15.7522 7.31553 14.9874 7.08172 14.3715 7.4092C13.7556 7.73669 13.5218 8.50147 13.8493 9.11738C14.0038 9.40809 14.4944 8.9354 14.7822 9.03703Z" fill="#3A3B45"/>
+<path d="M8.83422 9.03703C8.5123 9.1507 8.38422 9.81254 8.05901 9.6396C7.4431 9.31212 7.20928 8.54734 7.53676 7.93142C7.86425 7.31553 8.62903 7.08172 9.24494 7.4092C9.86086 7.73669 10.0947 8.50147 9.76719 9.11738C9.61262 9.40809 9.122 8.9354 8.83422 9.03703Z" fill="#3A3B45"/>
+<path d="M11.8679 15.1044C14.3507 15.1044 15.1519 12.8908 15.1519 11.7541C15.1519 11.1633 14.7547 11.3492 14.1187 11.6641C13.5309 11.9551 12.739 12.3563 11.8679 12.3563C10.0543 12.3563 8.58398 10.6173 8.58398 11.7541C8.58398 12.8908 9.38514 15.1044 11.8679 15.1044Z" fill="#3A3B45"/>
+<mask id="mask0_8587_60183" style="mask-type:alpha" maskUnits="userSpaceOnUse" x="8" y="11" width="8" height="5">
+<path d="M11.8562 15.1005C14.339 15.1005 15.1402 12.8869 15.1402 11.7502C15.1402 11.1594 14.743 11.3453 14.1069 11.6602C13.5191 11.9512 12.7273 12.3524 11.8562 12.3524C10.0425 12.3524 8.57227 10.6134 8.57227 11.7502C8.57227 12.8869 9.37342 15.1005 11.8562 15.1005Z" fill="white"/>
+</mask>
+<g mask="url(#mask0_8587_60183)">
+<path d="M11.9194 17.6824C13.1294 17.6824 14.1103 16.7016 14.1103 15.4916C14.1103 14.5491 13.5152 13.7457 12.6803 13.4364C12.6496 13.425 12.6185 13.4143 12.5872 13.4043C12.3766 13.337 12.1523 14.0606 11.9194 14.0606C11.7018 14.0606 11.4917 13.3324 11.2933 13.3915C10.3884 13.6609 9.72852 14.4991 9.72852 15.4916C9.72852 16.7016 10.7094 17.6824 11.9194 17.6824Z" fill="#F94040"/>
+</g>
+<path d="M17.8698 10.2273C18.3232 10.2273 18.6908 9.85972 18.6908 9.40631C18.6908 8.9529 18.3232 8.58533 17.8698 8.58533C17.4164 8.58533 17.0488 8.9529 17.0488 9.40631C17.0488 9.85972 17.4164 10.2273 17.8698 10.2273Z" fill="#FF9D0B"/>
+<path d="M6.11981 10.2273C6.57323 10.2273 6.9408 9.85972 6.9408 9.40631C6.9408 8.9529 6.57323 8.58533 6.11981 8.58533C5.66638 8.58533 5.29883 8.9529 5.29883 9.40631C5.29883 9.85972 5.66638 10.2273 6.11981 10.2273Z" fill="#FF9D0B"/>
+<path d="M4.42915 13.0092C4.02018 13.0092 3.65465 13.1771 3.39976 13.4818C3.24214 13.6705 3.07743 13.9746 3.06404 14.4301C2.89252 14.3808 2.72757 14.3533 2.57347 14.3533C2.18193 14.3533 1.82827 14.5033 1.57819 14.7759C1.25687 15.1258 1.11414 15.5557 1.17628 15.9859C1.20584 16.1908 1.2743 16.3744 1.3766 16.5444C1.16087 16.719 1.00198 16.962 0.925188 17.2543C0.865067 17.4834 0.803429 17.9606 1.12526 18.4522C1.10479 18.4842 1.0856 18.5176 1.06766 18.5517C0.874161 18.919 0.861783 19.334 1.03255 19.7205C1.29147 20.3063 1.93487 20.7678 3.18429 21.2632C3.96157 21.5714 4.67267 21.7684 4.67899 21.7702C5.70661 22.0367 6.63596 22.1721 7.44053 22.1721C8.91931 22.1721 9.97801 21.7192 10.5873 20.8259C11.5679 19.3876 11.4277 18.072 10.1589 16.8039C9.45662 16.1021 8.98979 15.0674 8.89254 14.8403C8.69651 14.1679 8.17815 13.4204 7.3165 13.4204C7.244 13.4204 7.17049 13.4262 7.09824 13.4376C6.72084 13.4969 6.39093 13.7142 6.15525 14.0411C5.90087 13.7248 5.65381 13.4732 5.43025 13.3312C5.09327 13.1175 4.75654 13.0092 4.42915 13.0092ZM4.42915 14.0196C4.55799 14.0196 4.71536 14.0744 4.88891 14.1846C5.42773 14.5263 6.46747 16.3136 6.84816 17.0087C6.97573 17.2417 7.19373 17.3402 7.39001 17.3402C7.77953 17.3402 8.08368 16.9529 7.42563 16.4608C6.43615 15.7204 6.78324 14.5102 7.25562 14.4356C7.27633 14.4324 7.29679 14.4308 7.3165 14.4308C7.74594 14.4308 7.93539 15.171 7.93539 15.171C7.93539 15.171 8.49063 16.5654 9.44449 17.5185C10.3984 18.4719 10.4476 19.237 9.75243 20.2566C9.27828 20.9517 8.37064 21.1617 7.44053 21.1617C6.47581 21.1617 5.48684 20.9358 4.93261 20.7921C4.90533 20.785 1.53474 19.8329 1.96165 19.0226C2.03339 18.8864 2.15161 18.8318 2.3004 18.8318C2.90162 18.8318 3.99517 19.7266 4.46528 19.7266C4.57036 19.7266 4.64438 19.6819 4.67469 19.5727C4.87501 18.8541 1.62896 18.5519 1.90254 17.5109C1.95079 17.3268 2.08164 17.252 2.26554 17.2523C3.06 17.2523 4.84243 18.6495 5.21604 18.6495C5.24458 18.6495 5.26504 18.6411 5.27616 18.6234C5.46334 18.3213 5.36078 18.1104 4.0414 17.3119C2.72201 16.5131 1.79594 16.0327 2.32263 15.4592C2.38326 15.393 2.46915 15.3637 2.57347 15.3637C3.3745 15.364 5.26706 17.0863 5.26706 17.0863C5.26706 17.0863 5.77784 17.6175 6.08679 17.6175C6.15777 17.6175 6.21814 17.5895 6.25907 17.5203C6.47808 17.151 4.22479 15.4433 4.09773 14.7388C4.01159 14.2613 4.1581 14.0196 4.42915 14.0196Z" fill="#FF9D0B"/>
+<path d="M9.75883 20.2539C10.454 19.2344 10.4048 18.4692 9.4509 17.5159C8.49704 16.5628 7.9418 15.1684 7.9418 15.1684C7.9418 15.1684 7.73441 14.3585 7.26203 14.433C6.78964 14.5075 6.44281 15.7178 7.43228 16.4582C8.42176 17.1984 7.23525 17.7013 6.85456 17.0061C6.47388 16.3109 5.43438 14.5237 4.89531 14.1819C4.35649 13.8402 3.97707 14.0316 4.10414 14.7362C4.2312 15.4407 6.48474 17.1483 6.26547 17.5179C6.04621 17.8872 5.27347 17.0837 5.27347 17.0837C5.27347 17.0837 2.85548 14.8832 2.32903 15.4566C1.80258 16.03 2.72842 16.5105 4.0478 17.3093C5.36744 18.1078 5.46975 18.3187 5.28257 18.6208C5.09513 18.9229 2.18251 16.4673 1.90893 17.5083C1.63561 18.5493 4.88142 18.8514 4.6811 19.5701C4.48078 20.2891 2.3947 18.2098 1.96804 19.0199C1.54113 19.8303 4.91173 20.7824 4.93901 20.7895C6.02777 21.0719 8.79285 21.6703 9.75883 20.2539Z" fill="#FFD21E"/>
+<path d="M19.5568 13.0092C19.9658 13.0092 20.3313 13.1771 20.5862 13.4818C20.7439 13.6705 20.9086 13.9746 20.9219 14.4301C21.0935 14.3808 21.2584 14.3533 21.4125 14.3533C21.8041 14.3533 22.1577 14.5033 22.4078 14.7759C22.7291 15.1258 22.8718 15.5557 22.8097 15.9859C22.7802 16.1908 22.7117 16.3744 22.6094 16.5444C22.8251 16.719 22.984 16.962 23.0608 17.2543C23.1209 17.4834 23.1826 17.9606 22.8607 18.4522C22.8812 18.4842 22.9004 18.5176 22.9183 18.5517C23.1118 18.919 23.1242 19.334 22.9534 19.7205C22.6945 20.3063 22.0511 20.7678 20.8017 21.2632C20.0244 21.5714 19.3133 21.7684 19.307 21.7702C18.2794 22.0367 17.35 22.1721 16.5455 22.1721C15.0667 22.1721 14.008 21.7192 13.3987 20.8259C12.418 19.3876 12.5582 18.072 13.8271 16.8039C14.5294 16.1021 14.9962 15.0674 15.0935 14.8403C15.2895 14.1679 15.8078 13.4204 16.6695 13.4204C16.742 13.4204 16.8155 13.4262 16.8877 13.4376C17.2651 13.4969 17.5951 13.7142 17.8307 14.0411C18.0851 13.7248 18.3322 13.4732 18.5557 13.3312C18.8927 13.1175 19.2295 13.0092 19.5568 13.0092ZM19.5568 14.0196C19.428 14.0196 19.2706 14.0744 19.0971 14.1846C18.5583 14.5263 17.5185 16.3136 17.1378 17.0087C17.0103 17.2417 16.7923 17.3402 16.596 17.3402C16.2065 17.3402 15.9023 16.9529 16.5604 16.4608C17.5498 15.7204 17.2028 14.5102 16.7304 14.4356C16.7097 14.4324 16.6892 14.4308 16.6695 14.4308C16.2401 14.4308 16.0506 15.171 16.0506 15.171C16.0506 15.171 15.4954 16.5654 14.5415 17.5185C13.5876 18.4719 13.5384 19.237 14.2336 20.2566C14.7077 20.9517 15.6153 21.1617 16.5455 21.1617C17.5102 21.1617 18.4992 20.9358 19.0534 20.7921C19.0807 20.785 22.4513 19.8329 22.0243 19.0226C21.9526 18.8864 21.8344 18.8318 21.6856 18.8318C21.0844 18.8318 19.9908 19.7266 19.5207 19.7266C19.4156 19.7266 19.3416 19.6819 19.3113 19.5727C19.111 18.8541 22.357 18.5519 22.0835 17.5109C22.0352 17.3268 21.9043 17.252 21.7204 17.2523C20.926 17.2523 19.1436 18.6495 18.77 18.6495C18.7414 18.6495 18.7209 18.6411 18.7098 18.6234C18.5226 18.3213 18.6252 18.1104 19.9446 17.3119C21.264 16.5131 22.1901 16.0327 21.6634 15.4592C21.6027 15.393 21.5168 15.3637 21.4125 15.3637C20.6115 15.364 18.7189 17.0863 18.7189 17.0863C18.7189 17.0863 18.2081 17.6175 17.8992 17.6175C17.8282 17.6175 17.7678 17.5895 17.7269 17.5203C17.5079 17.151 19.7612 15.4433 19.8883 14.7388C19.9744 14.2613 19.8279 14.0196 19.5568 14.0196Z" fill="#FF9D0B"/>
+<path d="M14.2354 20.2539C13.5402 19.2344 13.5895 18.4692 14.5433 17.5159C15.4972 16.5628 16.0524 15.1684 16.0524 15.1684C16.0524 15.1684 16.2598 14.3585 16.7322 14.433C17.2046 14.5075 17.5514 15.7178 16.5619 16.4582C15.5724 17.1984 16.759 17.7013 17.1396 17.0061C17.5203 16.3109 18.5598 14.5237 19.0989 14.1819C19.6377 13.8402 20.0171 14.0316 19.8901 14.7362C19.763 15.4407 17.5095 17.1483 17.7287 17.5179C17.948 17.8872 18.7207 17.0837 18.7207 17.0837C18.7207 17.0837 21.1387 14.8832 21.6652 15.4566C22.1916 16.03 21.2658 16.5105 19.9464 17.3093C18.6268 18.1078 18.5245 18.3187 18.7116 18.6208C18.8991 18.9229 21.8117 16.4673 22.0853 17.5083C22.3586 18.5493 19.1128 18.8514 19.3131 19.5701C19.5134 20.2891 21.5995 18.2098 22.0262 19.0199C22.4531 19.8303 19.0825 20.7824 19.0552 20.7895C17.9664 21.0719 15.2014 21.6703 14.2354 20.2539Z" fill="#FFD21E"/>
+</svg>

api/core/model_runtime/model_providers/huggingface_hub/_common.py

@@ -0,0 +1,15 @@
+from huggingface_hub.utils import HfHubHTTPError, BadRequestError
+
+from core.model_runtime.errors.invoke import InvokeBadRequestError, InvokeError
+
+
+class _CommonHuggingfaceHub:
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        return {
+            InvokeBadRequestError: [
+                HfHubHTTPError,
+                BadRequestError
+            ]
+        }

api/core/model_runtime/model_providers/huggingface_hub/huggingface_hub.py

@@ -0,0 +1,11 @@
+import logging
+
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class HuggingfaceHubProvider(ModelProvider):
+
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        pass

api/core/model_runtime/model_providers/huggingface_hub/huggingface_hub.yaml

@@ -0,0 +1,102 @@
+provider: huggingface_hub
+label:
+  en_US: Hugging Face Model
+icon_small:
+    en_US: icon_s_en.svg
+icon_large:
+    en_US: icon_l_en.svg
+background: "#FFF8DC"
+help:
+  title:
+    en_US: Get your API key from Hugging Face Hub
+    zh_Hans: 从 Hugging Face Hub 获取 API Key
+  url:
+    en_US: https://huggingface.co/settings/tokens
+supported_model_types:
+- llm
+- text-embedding
+configurate_methods:
+- customizable-model
+model_credential_schema:
+  model:
+    label:
+      en_US: Model Name
+      zh_Hans: 模型名称
+  credential_form_schemas:
+  - variable: huggingfacehub_api_type
+    label:
+      en_US: Endpoint Type
+      zh_Hans: 端点类型
+    type: radio
+    required: true
+    default: hosted_inference_api
+    options:
+    - value: hosted_inference_api
+      label:
+        en_US: Hosted Inference API
+    - value: inference_endpoints
+      label:
+        en_US: Inference Endpoints
+  - variable: huggingfacehub_api_token
+    label:
+      en_US: API Token
+      zh_Hans: API Token
+    type: secret-input
+    required: true
+    placeholder:
+      en_US: Enter your Hugging Face Hub API Token here
+      zh_Hans: 在此输入您的 Hugging Face Hub API Token
+  - variable: huggingface_namespace
+    label:
+      en_US: 'User Name / Organization Name'
+      zh_Hans: '用户名 / 组织名称'
+    type: text-input
+    required: true
+    placeholder:
+      en_US: 'Enter your User Name / Organization Name here'
+      zh_Hans: '在此输入您的用户名 / 组织名称'
+    show_on:
+    - variable: __model_type
+      value: text-embedding
+    - variable: huggingfacehub_api_type
+      value: inference_endpoints
+  - variable: huggingfacehub_endpoint_url
+    label:
+      en_US: Endpoint URL
+      zh_Hans: 端点 URL
+    type: text-input
+    required: true
+    placeholder:
+      en_US: Enter your Endpoint URL here
+      zh_Hans: 在此输入您的端点 URL
+    show_on:
+    - variable: huggingfacehub_api_type
+      value: inference_endpoints
+  - variable: task_type
+    label:
+      en_US: Task
+      zh_Hans: Task
+    type: select
+    options:
+    - value: text2text-generation
+      label:
+        en_US: Text-to-Text Generation
+      show_on:
+      - variable: __model_type
+        value: llm
+    - value: text-generation
+      label:
+        en_US: Text Generation
+        zh_Hans: 文本生成
+      show_on:
+      - variable: __model_type
+        value: llm
+    - value: feature-extraction
+      label:
+        en_US: Feature Extraction
+      show_on:
+      - variable: __model_type
+        value: text-embedding
+    show_on:
+    - variable: huggingfacehub_api_type
+      value: inference_endpoints

api/core/model_runtime/model_providers/huggingface_hub/llm/llm.py

@@ -0,0 +1,241 @@
+from typing import Optional, List, Union, Generator
+
+from huggingface_hub import InferenceClient
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import BadRequestError
+
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.defaults import PARAMETER_RULE_TEMPLATE
+from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMMode
+from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool, AssistantPromptMessage, \
+    UserPromptMessage, SystemPromptMessage
+from core.model_runtime.entities.model_entities import ParameterRule, DefaultParameterName, AIModelEntity, ModelType, \
+    FetchFrom
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
+from core.model_runtime.model_providers.huggingface_hub._common import _CommonHuggingfaceHub
+
+
+class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel):
+    def _invoke(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], model_parameters: dict,
+                tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True,
+                user: Optional[str] = None) -> Union[LLMResult, Generator]:
+
+        client = InferenceClient(token=credentials['huggingfacehub_api_token'])
+
+        if credentials['huggingfacehub_api_type'] == 'inference_endpoints':
+            model = credentials['huggingfacehub_endpoint_url']
+
+        if 'baichuan' in model.lower():
+            stream = False
+
+        response = client.text_generation(
+            prompt=prompt_messages[0].content,
+            details=True,
+            stream=stream,
+            model=model,
+            stop_sequences=stop,
+            **model_parameters)
+
+        if stream:
+            return self._handle_generate_stream_response(model, credentials, prompt_messages, response)
+
+        return self._handle_generate_response(model, credentials, prompt_messages, response)
+
+    def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage],
+                       tools: Optional[list[PromptMessageTool]] = None) -> int:
+        prompt = self._convert_messages_to_prompt(prompt_messages)
+        return self._get_num_tokens_by_gpt2(prompt)
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        try:
+            if 'huggingfacehub_api_type' not in credentials:
+                raise CredentialsValidateFailedError('Huggingface Hub Endpoint Type must be provided.')
+
+            if credentials['huggingfacehub_api_type'] not in ('inference_endpoints', 'hosted_inference_api'):
+                raise CredentialsValidateFailedError('Huggingface Hub Endpoint Type is invalid.')
+
+            if 'huggingfacehub_api_token' not in credentials:
+                raise CredentialsValidateFailedError('Huggingface Hub Access Token must be provided.')
+
+            if credentials['huggingfacehub_api_type'] == 'inference_endpoints':
+                if 'huggingfacehub_endpoint_url' not in credentials:
+                    raise CredentialsValidateFailedError('Huggingface Hub Endpoint URL must be provided.')
+
+                if 'task_type' not in credentials:
+                    raise CredentialsValidateFailedError('Huggingface Hub Task Type must be provided.')
+            elif credentials['huggingfacehub_api_type'] == 'hosted_inference_api':
+                credentials['task_type'] = self._get_hosted_model_task_type(credentials['huggingfacehub_api_token'],
+                                                                            model)
+
+            if credentials['task_type'] not in ("text2text-generation", "text-generation"):
+                raise CredentialsValidateFailedError('Huggingface Hub Task Type must be one of text2text-generation, '
+                                                     'text-generation.')
+
+            client = InferenceClient(token=credentials['huggingfacehub_api_token'])
+
+            if credentials['huggingfacehub_api_type'] == 'inference_endpoints':
+                model = credentials['huggingfacehub_endpoint_url']
+
+            try:
+                client.text_generation(
+                    prompt='Who are you?',
+                    stream=True,
+                    model=model)
+            except BadRequestError as e:
+                raise CredentialsValidateFailedError('Only available for models running on with the `text-generation-inference`. '
+                                                     'To learn more about the TGI project, please refer to https://github.com/huggingface/text-generation-inference.')
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        entity = AIModelEntity(
+            model=model,
+            label=I18nObject(
+                en_US=model
+            ),
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_type=ModelType.LLM,
+            model_properties={
+                'mode': LLMMode.COMPLETION.value
+            },
+            parameter_rules=self._get_customizable_model_parameter_rules()
+        )
+
+        return entity
+
+    @staticmethod
+    def _get_customizable_model_parameter_rules() -> list[ParameterRule]:
+        temperature_rule_dict = PARAMETER_RULE_TEMPLATE.get(
+            DefaultParameterName.TEMPERATURE).copy()
+        temperature_rule_dict['name'] = 'temperature'
+        temperature_rule = ParameterRule(**temperature_rule_dict)
+        temperature_rule.default = 0.5
+
+        top_p_rule_dict = PARAMETER_RULE_TEMPLATE.get(DefaultParameterName.TOP_P).copy()
+        top_p_rule_dict['name'] = 'top_p'
+        top_p_rule = ParameterRule(**top_p_rule_dict)
+        top_p_rule.default = 0.5
+
+        top_k_rule = ParameterRule(
+            name='top_k',
+            label={
+                'en_US': 'Top K',
+                'zh_Hans': 'Top K',
+            },
+            type='int',
+            help={
+                'en_US': 'The number of highest probability vocabulary tokens to keep for top-k-filtering.',
+                'zh_Hans': '保留的最高概率词汇标记的数量。',
+            },
+            required=False,
+            default=2,
+            min=1,
+            max=10,
+            precision=0,
+        )
+
+        return [temperature_rule, top_k_rule, top_p_rule]
+
+    def _handle_generate_stream_response(self,
+                                         model: str,
+                                         credentials: dict,
+                                         prompt_messages: list[PromptMessage],
+                                         response: Generator) -> Generator:
+        index = -1
+        for chunk in response:
+            # skip special tokens
+            if chunk.token.special:
+                continue
+
+            index += 1
+
+            assistant_prompt_message = AssistantPromptMessage(
+                content=chunk.token.text
+            )
+
+            prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+            completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+            usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+            yield LLMResultChunk(
+                model=model,
+                prompt_messages=prompt_messages,
+                delta=LLMResultChunkDelta(
+                    index=index,
+                    message=assistant_prompt_message,
+                    usage=usage,
+                ),
+            )
+
+    def _handle_generate_response(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], response: any) -> LLMResult:
+        if isinstance(response, str):
+            content = response
+        else:
+            content = response.generated_text
+
+        assistant_prompt_message = AssistantPromptMessage(
+            content=content
+        )
+
+        prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
+        completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])
+
+        usage = self._calc_response_usage(model, credentials, prompt_tokens, completion_tokens)
+
+        result = LLMResult(
+            model=model,
+            prompt_messages=prompt_messages,
+            message=assistant_prompt_message,
+            usage=usage,
+        )
+        return result
+
+    @staticmethod
+    def _get_hosted_model_task_type(huggingfacehub_api_token: str, model_name: str):
+        hf_api = HfApi(token=huggingfacehub_api_token)
+        model_info = hf_api.model_info(repo_id=model_name)
+
+        try:
+            if not model_info:
+                raise ValueError(f'Model {model_name} not found.')
+
+            if 'inference' in model_info.cardData and not model_info.cardData['inference']:
+                raise ValueError(f'Inference API has been turned off for this model {model_name}.')
+
+            valid_tasks = ("text2text-generation", "text-generation")
+            if model_info.pipeline_tag not in valid_tasks:
+                raise ValueError(f"Model {model_name} is not a valid task, "
+                                 f"must be one of {valid_tasks}.")
+        except Exception as e:
+            raise CredentialsValidateFailedError(f"{str(e)}")
+
+        return model_info.pipeline_tag
+
+    def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str:
+        messages = messages.copy()  # don't mutate the original list
+
+        text = "".join(
+            self._convert_one_message_to_text(message)
+            for message in messages
+        )
+
+        return text.rstrip()
+
+    @staticmethod
+    def _convert_one_message_to_text(message: PromptMessage) -> str:
+        human_prompt = "\n\nHuman:"
+        ai_prompt = "\n\nAssistant:"
+        content = message.content
+
+        if isinstance(message, UserPromptMessage):
+            message_text = f"{human_prompt} {content}"
+        elif isinstance(message, AssistantPromptMessage):
+            message_text = f"{ai_prompt} {content}"
+        elif isinstance(message, SystemPromptMessage):
+            message_text = content
+        else:
+            raise ValueError(f"Got unknown type {message}")
+
+        return message_text

api/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py

@@ -0,0 +1,191 @@
+import json
+import time
+from typing import Optional
+
+import numpy as np
+import requests
+from huggingface_hub import InferenceClient, HfApi
+
+from core.model_runtime.entities.common_entities import I18nObject
+from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType, PriceType
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
+from core.model_runtime.model_providers.huggingface_hub._common import _CommonHuggingfaceHub
+
+
+HUGGINGFACE_ENDPOINT_API = 'https://api.endpoints.huggingface.cloud/v2/endpoint/'
+
+
+class HuggingfaceHubTextEmbeddingModel(_CommonHuggingfaceHub, TextEmbeddingModel):
+
+    def _invoke(self, model: str, credentials: dict, texts: list[str],
+                user: Optional[str] = None) -> TextEmbeddingResult:
+        client = InferenceClient(token=credentials['huggingfacehub_api_token'])
+
+        execute_model = model
+
+        if credentials['huggingfacehub_api_type'] == 'inference_endpoints':
+            execute_model = credentials['huggingfacehub_endpoint_url']
+
+        output = client.post(
+            json={
+                "inputs": texts,
+                "options": {
+                    "wait_for_model": False,
+                    "use_cache": False
+                }
+            },
+            model=execute_model)
+
+        embeddings = json.loads(output.decode())
+
+        tokens = self.get_num_tokens(model, credentials, texts)
+        usage = self._calc_response_usage(model, credentials, tokens)
+
+        return TextEmbeddingResult(
+            embeddings=self._mean_pooling(embeddings),
+            usage=usage,
+            model=model
+        )
+
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        num_tokens = 0
+        for text in texts:
+            num_tokens += self._get_num_tokens_by_gpt2(text)
+        return num_tokens
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        try:
+            if 'huggingfacehub_api_type' not in credentials:
+                raise CredentialsValidateFailedError('Huggingface Hub Endpoint Type must be provided.')
+
+            if 'huggingfacehub_api_token' not in credentials:
+                raise CredentialsValidateFailedError('Huggingface Hub API Token must be provided.')
+
+            if credentials['huggingfacehub_api_type'] == 'inference_endpoints':
+                if 'huggingface_namespace' not in credentials:
+                    raise CredentialsValidateFailedError('Huggingface Hub User Name / Organization Name must be provided.')
+
+                if 'huggingfacehub_endpoint_url' not in credentials:
+                    raise CredentialsValidateFailedError('Huggingface Hub Endpoint URL must be provided.')
+
+                if 'task_type' not in credentials:
+                    raise CredentialsValidateFailedError('Huggingface Hub Task Type must be provided.')
+
+                if credentials['task_type'] != 'feature-extraction':
+                    raise CredentialsValidateFailedError('Huggingface Hub Task Type is invalid.')
+
+                self._check_endpoint_url_model_repository_name(credentials, model)
+
+                model = credentials['huggingfacehub_endpoint_url']
+
+            elif credentials['huggingfacehub_api_type'] == 'hosted_inference_api':
+                self._check_hosted_model_task_type(credentials['huggingfacehub_api_token'],
+                                                   model)
+            else:
+                raise CredentialsValidateFailedError('Huggingface Hub Endpoint Type is invalid.')
+
+            client = InferenceClient(token=credentials['huggingfacehub_api_token'])
+            client.feature_extraction(text='hello world', model=model)
+        except Exception as ex:
+            raise CredentialsValidateFailedError(str(ex))
+
+    def get_customizable_model_schema(self, model: str, credentials: dict) -> Optional[AIModelEntity]:
+        entity = AIModelEntity(
+            model=model,
+            label=I18nObject(
+                en_US=model
+            ),
+            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
+            model_type=ModelType.TEXT_EMBEDDING,
+            model_properties={
+                'context_size': 10000,
+                'max_chunks': 1
+            }
+        )
+        return entity
+
+    # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task
+    # Returned values are a list of floats, or a list[list[floats]]
+    # (depending on if you sent a string or a list of string,
+    # and if the automatic reduction, usually mean_pooling for instance was applied for you or not.
+    # This should be explained on the model's README.)
+    @staticmethod
+    def _mean_pooling(embeddings: list) -> list[float]:
+        # If automatic reduction by giving model, no need to mean_pooling.
+        # For example one: List[List[float]]
+        if not isinstance(embeddings[0][0], list):
+            return embeddings
+
+        # For example two: List[List[List[float]]], need to mean_pooling.
+        sentence_embeddings = [np.mean(embedding[0], axis=0).tolist() for embedding in embeddings]
+        return sentence_embeddings
+
+    @staticmethod
+    def _check_hosted_model_task_type(huggingfacehub_api_token: str, model_name: str) -> None:
+        hf_api = HfApi(token=huggingfacehub_api_token)
+        model_info = hf_api.model_info(repo_id=model_name)
+
+        try:
+            if not model_info:
+                raise ValueError(f'Model {model_name} not found.')
+
+            if 'inference' in model_info.cardData and not model_info.cardData['inference']:
+                raise ValueError(f'Inference API has been turned off for this model {model_name}.')
+
+            valid_tasks = "feature-extraction"
+            if model_info.pipeline_tag not in valid_tasks:
+                raise ValueError(f"Model {model_name} is not a valid task, "
+                                 f"must be one of {valid_tasks}.")
+        except Exception as e:
+            raise CredentialsValidateFailedError(f"{str(e)}")
+
+    def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
+        input_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=tokens
+        )
+
+        # transform usage
+        usage = EmbeddingUsage(
+            tokens=tokens,
+            total_tokens=tokens,
+            unit_price=input_price_info.unit_price,
+            price_unit=input_price_info.unit,
+            total_price=input_price_info.total_amount,
+            currency=input_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage
+
+    @staticmethod
+    def _check_endpoint_url_model_repository_name(credentials: dict, model_name: str):
+        try:
+            url = f'{HUGGINGFACE_ENDPOINT_API}{credentials["huggingface_namespace"]}'
+            headers = {
+                'Authorization': f'Bearer {credentials["huggingfacehub_api_token"]}',
+                'Content-Type': 'application/json'
+            }
+
+            response = requests.get(url=url, headers=headers)
+
+            if response.status_code != 200:
+                raise ValueError('User Name or Organization Name is invalid.')
+
+            model_repository_name = ''
+
+            for item in response.json().get("items", []):
+                if item.get("status", {}).get("url") == credentials['huggingfacehub_endpoint_url']:
+                    model_repository_name = item.get("model", {}).get("repository")
+                    break
+
+            if model_repository_name != model_name:
+                raise ValueError(
+                    f'Model Name {model_name} is invalid. Please check it on the inference endpoints console.')
+
+        except Exception as e:
+            raise ValueError(str(e))

api/core/model_runtime/model_providers/jina/_assets/icon_l_en.svg

@@ -0,0 +1,12 @@
+<svg width="58" height="24" viewBox="0 0 58 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_13814_61529)">
+<path d="M4.47132 23.952C6.49932 23.952 8.14332 22.308 8.14332 20.28C8.14332 18.252 6.49932 16.608 4.47132 16.608C2.44332 16.608 0.799316 18.252 0.799316 20.28C0.799316 22.308 2.44332 23.952 4.47132 23.952Z" fill="#EB6161"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M16.0387 8.71204C16.5187 8.71204 16.9027 9.09604 16.9027 9.57604L16.8547 16.608C16.8547 20.616 13.6387 23.88 9.63074 23.952H9.51074V16.632H9.53474L9.55874 9.60004C9.55874 9.12004 9.94274 8.73604 10.4227 8.73604H16.0387V8.71204ZM27.3187 8.71204C27.7987 8.71204 28.1827 9.09604 28.1827 9.57604V19.416C28.1827 19.896 27.7987 20.28 27.3187 20.28H21.7027C21.2227 20.28 20.8387 19.896 20.8387 19.416V9.57604C20.8387 9.09604 21.2227 8.71204 21.7027 8.71204H27.3187ZM36.1507 8.68804H36.2707C39.8707 8.73604 42.7987 11.64 42.8947 15.24V19.392C42.8947 19.872 42.5107 20.256 42.0307 20.256H32.9587C32.4787 20.256 32.0947 19.872 32.0947 19.392V9.55204C32.0947 9.07204 32.4787 8.68804 32.9587 8.68804H36.1507ZM51.0067 20.16C47.9827 19.968 45.5587 17.448 45.5587 14.376C45.5587 11.184 48.1507 8.59204 51.3427 8.59204C54.4147 8.59204 56.9347 10.992 57.1267 14.04V19.296C57.1267 19.776 56.7427 20.16 56.2627 20.16H51.0067Z" fill="#009191"/>
+<path d="M24.4987 7.344C26.5267 7.344 28.1707 5.7 28.1707 3.672C28.1707 1.644 26.5267 0 24.4987 0C22.4707 0 20.8267 1.644 20.8267 3.672C20.8267 5.7 22.4707 7.344 24.4987 7.344Z" fill="#FBCB67"/>
+</g>
+<defs>
+<clipPath id="clip0_13814_61529">
+<rect width="56.4" height="24" fill="white" transform="translate(0.800781)"/>
+</clipPath>
+</defs>
+</svg>

api/core/model_runtime/model_providers/jina/_assets/icon_s_en.svg

@@ -0,0 +1,4 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.56053 21.4486C9.07925 21.4486 11.1211 19.4068 11.1211 16.8882C11.1211 14.3696 9.07925 12.3279 6.56053 12.3279C4.04182 12.3279 2 14.3696 2 16.8882C2 19.4068 4.04182 21.4486 6.56053 21.4486Z" fill="#EB6161"/>
+<path d="M22.0002 3.59467L21.9406 12.3279C21.9406 17.3055 17.9464 21.3591 12.9685 21.4485L12.8789 12.3577L12.8791 3.62447C12.8791 3.02835 13.356 2.55145 13.9522 2.55145H20.9271C21.5233 2.55145 22.0002 2.99854 22.0002 3.59467Z" fill="#009191"/>
+</svg>

api/core/model_runtime/model_providers/jina/jina.py

@@ -0,0 +1,32 @@
+import logging
+
+from core.model_runtime.entities.model_entities import ModelType
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.model_provider import ModelProvider
+
+logger = logging.getLogger(__name__)
+
+
+class JinaProvider(ModelProvider):
+
+    def validate_provider_credentials(self, credentials: dict) -> None:
+        """
+        Validate provider credentials
+        if validate failed, raise exception
+
+        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
+        """
+        try:
+            model_instance = self.get_model_instance(ModelType.TEXT_EMBEDDING)
+
+            # Use `jina-embeddings-v2-base-en` model for validate,
+            # no matter what model you pass in, text completion model or chat model
+            model_instance.validate_credentials(
+                model='jina-embeddings-v2-base-en',
+                credentials=credentials
+            )
+        except CredentialsValidateFailedError as ex:
+            raise ex
+        except Exception as ex:
+            logger.exception(f'{self.get_provider_schema().provider} credentials validate failed')
+            raise ex

api/core/model_runtime/model_providers/jina/jina.yaml

@@ -0,0 +1,30 @@
+provider: jina
+label:
+  en_US: Jina
+description:
+    en_US: Embedding Model Supported
+icon_small:
+  en_US: icon_s_en.svg
+icon_large:
+  en_US: icon_l_en.svg
+background: "#EFFDFD"
+help:
+  title:
+    en_US: Get your API key from Jina AI
+    zh_Hans: 从 Jina 获取 API Key
+  url:
+    en_US: https://jina.ai/embeddings/
+supported_model_types:
+- text-embedding
+configurate_methods:
+- predefined-model
+provider_credential_schema:
+  credential_form_schemas:
+  - variable: api_key
+    label:
+      en_US: API Key
+    type: secret-input
+    required: true
+    placeholder:
+      zh_Hans: 在此输入您的 API Key
+      en_US: Enter your API Key

api/core/model_runtime/model_providers/jina/text_embedding/jina-embeddings-v2-base-en.yaml

@@ -0,0 +1,9 @@
+model: jina-embeddings-v2-base-en
+model_type: text-embedding
+model_properties:
+  context_size: 8192
+  max_chunks: 2048
+pricing:
+  input: '0.001'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/jina/text_embedding/jina-embeddings-v2-small-en.yaml

@@ -0,0 +1,9 @@
+model: jina-embeddings-v2-small-en
+model_type: text-embedding
+model_properties:
+  context_size: 8192
+  max_chunks: 2048
+pricing:
+  input: '0.001'
+  unit: '0.001'
+  currency: USD

api/core/model_runtime/model_providers/jina/text_embedding/jina_tokenizer.py

@@ -0,0 +1,18 @@
+from transformers import AutoTokenizer
+from os.path import join, abspath, dirname
+
+class JinaTokenizer:
+    @staticmethod
+    def _get_num_tokens_by_jina_base(text: str) -> int:
+        """
+            use jina tokenizer to get num tokens
+        """
+        base_path = abspath(__file__)
+        gpt2_tokenizer_path = join(dirname(base_path), 'tokenizer')
+        tokenizer = AutoTokenizer.from_pretrained(gpt2_tokenizer_path)
+        tokens = tokenizer.encode(text)
+        return len(tokens)
+    
+    @staticmethod
+    def get_num_tokens(text: str) -> int:
+        return JinaTokenizer._get_num_tokens_by_jina_base(text)

api/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py

@@ -0,0 +1,166 @@
+from typing import Optional
+
+from core.model_runtime.entities.model_entities import PriceType
+from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel
+from core.model_runtime.errors.invoke import InvokeError, InvokeConnectionError, InvokeServerUnavailableError, \
+    InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError
+from core.model_runtime.model_providers.jina.text_embedding.jina_tokenizer import JinaTokenizer
+
+from requests import post
+from json import dumps, JSONDecodeError
+
+import time
+
+class JinaTextEmbeddingModel(TextEmbeddingModel):
+    """
+    Model class for Jina text embedding model.
+    """
+    api_base: str = 'https://api.jina.ai/v1/embeddings'
+    models: list[str] = ['jina-embeddings-v2-base-en', 'jina-embeddings-v2-small-en']
+
+    def _invoke(self, model: str, credentials: dict,
+                texts: list[str], user: Optional[str] = None) \
+            -> TextEmbeddingResult:
+        """
+        Invoke text embedding model
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :param user: unique user id
+        :return: embeddings result
+        """
+        api_key = credentials['api_key']
+        if model not in self.models:
+            raise InvokeBadRequestError('Invalid model name')
+        if not api_key:
+            raise CredentialsValidateFailedError('api_key is required')
+        url = self.api_base
+        headers = {
+            'Authorization': 'Bearer ' + api_key,
+            'Content-Type': 'application/json'
+        }
+
+        data = {
+            'model': model,
+            'input': texts
+        }
+
+        try:
+            response = post(url, headers=headers, data=dumps(data))
+        except Exception as e:
+            raise InvokeConnectionError(e)
+        
+        if response.status_code != 200:
+            try:
+                resp = response.json()
+                msg = resp['detail']
+                if response.status_code == 401:
+                    raise InvokeAuthorizationError(msg)
+                elif response.status_code == 429:
+                    raise InvokeRateLimitError(msg)
+                elif response.status_code == 500:
+                    raise InvokeServerUnavailableError(msg)
+                else:
+                    raise InvokeError(msg)
+            except JSONDecodeError as e:
+                raise InvokeServerUnavailableError(f"Failed to convert response to json: {e} with text: {response.text}")
+
+        try:
+            resp = response.json()
+            embeddings = resp['data']
+            usage = resp['usage']
+        except Exception as e:
+            raise InvokeServerUnavailableError(f"Failed to convert response to json: {e} with text: {response.text}")
+
+        usage = self._calc_response_usage(model=model, credentials=credentials, tokens=usage['total_tokens'])
+
+        result = TextEmbeddingResult(
+            model=model,
+            embeddings=[[
+                float(data) for data in x['embedding']
+            ] for x in embeddings],
+            usage=usage
+        )
+
+        return result
+
+    def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int:
+        """
+        Get number of tokens for given prompt messages
+
+        :param model: model name
+        :param credentials: model credentials
+        :param texts: texts to embed
+        :return:
+        """
+        num_tokens = 0
+        for text in texts:
+            # use JinaTokenizer to get num tokens
+            num_tokens += JinaTokenizer.get_num_tokens(text)
+        return num_tokens
+
+    def validate_credentials(self, model: str, credentials: dict) -> None:
+        """
+        Validate model credentials
+
+        :param model: model name
+        :param credentials: model credentials
+        :return:
+        """
+        try:
+            self._invoke(model=model, credentials=credentials, texts=['ping'])
+        except InvokeAuthorizationError:
+            raise CredentialsValidateFailedError('Invalid api key')
+
+    @property
+    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
+        return {
+            InvokeConnectionError: [
+                InvokeConnectionError
+            ],
+            InvokeServerUnavailableError: [
+                InvokeServerUnavailableError
+            ],
+            InvokeRateLimitError: [
+                InvokeRateLimitError
+            ],
+            InvokeAuthorizationError: [
+                InvokeAuthorizationError
+            ],
+            InvokeBadRequestError: [
+                KeyError
+            ]
+        }
+    
+    def _calc_response_usage(self, model: str, credentials: dict, tokens: int) -> EmbeddingUsage:
+        """
+        Calculate response usage
+
+        :param model: model name
+        :param credentials: model credentials
+        :param tokens: input tokens
+        :return: usage
+        """
+        # get input price info
+        input_price_info = self.get_price(
+            model=model,
+            credentials=credentials,
+            price_type=PriceType.INPUT,
+            tokens=tokens
+        )
+
+        # transform usage
+        usage = EmbeddingUsage(
+            tokens=tokens,
+            total_tokens=tokens,
+            unit_price=input_price_info.unit_price,
+            price_unit=input_price_info.unit,
+            total_price=input_price_info.total_amount,
+            currency=input_price_info.currency,
+            latency=time.perf_counter() - self.started_at
+        )
+
+        return usage