From a62ae5b7220f1459eb5b60198764eae93a489307 Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 14:13:26 +0000 Subject: [PATCH 1/6] Add LiteLLMComponent to the project --- .../langflow/components/llms/LiteLLM.py | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/backend/langflow/components/llms/LiteLLM.py diff --git a/src/backend/langflow/components/llms/LiteLLM.py b/src/backend/langflow/components/llms/LiteLLM.py new file mode 100644 index 000000000..6826bf1b2 --- /dev/null +++ b/src/backend/langflow/components/llms/LiteLLM.py @@ -0,0 +1,118 @@ +from dataclasses import Field +from langflow import CustomComponent +from typing import Optional, Union, Callable, Any, Dict +from langflow.field_typing import BaseLanguageModel +from langchain_community.chat_models import ChatLiteLLM +import os + + +class LiteLLMComponent(CustomComponent): + display_name = "LiteLLM" + description = "`LiteLLM` collection of large language models." + documentation = "https://python.langchain.com/docs/integrations/chat/litellm" + + def build_config(self): + return { + "model": { + "display_name": "Model name", + "field_type": "str", + "advanced": False, + "required": True, + "info": "The name of the model to use. For example, `gpt-3.5-turbo`.", + }, + "api_key": { + "display_name": "API key", + "field_type": "str", + "advanced": False, + "required": False, + "password": True, + }, + "streaming": { + "display_name": "Streaming", + "field_type": "bool", + "advanced": True, + "required": False, + "default": True, + }, + "temperature": { + "display_name": "Temperature", + "field_type": "float", + "advanced": True, + "required": False, + "default": 0.7, + }, + "model_kwargs": { + "display_name": "Model kwargs", + "field_type": "dict", + "advanced": True, + "required": False, + "default": {}, + }, + "top_p": { + "display_name": "Top p", + "field_type": "float", + "advanced": True, + "required": False, + }, + "top_k": { + "display_name": "Top k", + "field_type": "int", + "advanced": True, + "required": False, + }, + "n": { + "display_name": "N", + "field_type": "int", + "advanced": True, + "required": False, + "info": "Number of chat completions to generate for each prompt. " + "Note that the API may not return the full n completions if duplicates are generated.", + "default": 1, + }, + "max_tokens": { + "display_name": "Max tokens", + "field_type": "int", + "advanced": True, + "required": False, + "default": 256, + "info": "The maximum number of tokens to generate for each chat completion.", + }, + "max_retries": { + "display_name": "Max retries", + "field_type": "int", + "advanced": True, + "required": False, + "default": 6, + }, + } + + def build( + self, + model: str, + api_key: str, + streaming: bool = True, + temperature: Optional[float] = 0.7, + model_kwargs: Optional[Dict[str, Any]] = {}, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + n: Optional[int] = 1, + max_tokens: Optional[int] = 256, + max_retries: Optional[int] = 6, + ) -> Union[BaseLanguageModel, Callable]: + if api_key: + if "perplexity" in model: + os.environ["PERPLEXITYAI_API_KEY"] = api_key + elif "replicate" in model: + os.environ["REPLICATE_API_KEY"] = api_key + LLM = ChatLiteLLM( + model=model, + streaming=streaming, + temperature=temperature, + model_kwargs=model_kwargs, + top_p=top_p, + top_k=top_k, + n=n, + max_tokens=max_tokens, + max_retries=max_retries, + ) + return LLM From 32012df810a522e8ab2d672ff904c249ea437f40 Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 14:34:37 +0000 Subject: [PATCH 2/6] Add ChatLiteLLM component to backend --- .../langflow/components/llms/{LiteLLM.py => ChatLiteLLM.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename src/backend/langflow/components/llms/{LiteLLM.py => ChatLiteLLM.py} (97%) diff --git a/src/backend/langflow/components/llms/LiteLLM.py b/src/backend/langflow/components/llms/ChatLiteLLM.py similarity index 97% rename from src/backend/langflow/components/llms/LiteLLM.py rename to src/backend/langflow/components/llms/ChatLiteLLM.py index 6826bf1b2..25f75210b 100644 --- a/src/backend/langflow/components/llms/LiteLLM.py +++ b/src/backend/langflow/components/llms/ChatLiteLLM.py @@ -7,7 +7,7 @@ import os class LiteLLMComponent(CustomComponent): - display_name = "LiteLLM" + display_name = "ChatLiteLLM" description = "`LiteLLM` collection of large language models." documentation = "https://python.langchain.com/docs/integrations/chat/litellm" @@ -37,7 +37,7 @@ class LiteLLMComponent(CustomComponent): "temperature": { "display_name": "Temperature", "field_type": "float", - "advanced": True, + "advanced": False, "required": False, "default": 0.7, }, @@ -72,7 +72,7 @@ class LiteLLMComponent(CustomComponent): "max_tokens": { "display_name": "Max tokens", "field_type": "int", - "advanced": True, + "advanced": False, "required": False, "default": 256, "info": "The maximum number of tokens to generate for each chat completion.", From 4aadd27db160e3ea34bb5292543e0039f29f4c9c Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 15:36:04 +0000 Subject: [PATCH 3/6] Update ChatLiteLLM import and add verbose option --- .../langflow/components/llms/ChatLiteLLM.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/components/llms/ChatLiteLLM.py b/src/backend/langflow/components/llms/ChatLiteLLM.py index 25f75210b..07af297cf 100644 --- a/src/backend/langflow/components/llms/ChatLiteLLM.py +++ b/src/backend/langflow/components/llms/ChatLiteLLM.py @@ -2,7 +2,7 @@ from dataclasses import Field from langflow import CustomComponent from typing import Optional, Union, Callable, Any, Dict from langflow.field_typing import BaseLanguageModel -from langchain_community.chat_models import ChatLiteLLM +from langchain_community.chat_models.litellm import ChatLiteLLM, ChatLiteLLMException import os @@ -84,6 +84,13 @@ class LiteLLMComponent(CustomComponent): "required": False, "default": 6, }, + "verbose": { + "display_name": "Verbose", + "field_type": "bool", + "advanced": True, + "required": False, + "default": False, + }, } def build( @@ -98,12 +105,23 @@ class LiteLLMComponent(CustomComponent): n: Optional[int] = 1, max_tokens: Optional[int] = 256, max_retries: Optional[int] = 6, + verbose: Optional[bool] = False, ) -> Union[BaseLanguageModel, Callable]: + try: + import litellm + except ImportError: + raise ChatLiteLLMException( + "Could not import litellm python package. " + "Please install it with `pip install litellm`" + ) + litellm.drop_params=True + litellm.set_verbose=verbose if api_key: if "perplexity" in model: os.environ["PERPLEXITYAI_API_KEY"] = api_key elif "replicate" in model: os.environ["REPLICATE_API_KEY"] = api_key + LLM = ChatLiteLLM( model=model, streaming=streaming, @@ -116,3 +134,4 @@ class LiteLLMComponent(CustomComponent): max_retries=max_retries, ) return LLM +# litellm.drop_params=True \ No newline at end of file From 868aca053eeab66faf4dcbcc2dd4fb1242f0635a Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 15:50:54 +0000 Subject: [PATCH 4/6] Remove unused code in ChatLiteLLM.py --- src/backend/langflow/components/llms/ChatLiteLLM.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backend/langflow/components/llms/ChatLiteLLM.py b/src/backend/langflow/components/llms/ChatLiteLLM.py index 07af297cf..01b831bae 100644 --- a/src/backend/langflow/components/llms/ChatLiteLLM.py +++ b/src/backend/langflow/components/llms/ChatLiteLLM.py @@ -134,4 +134,3 @@ class LiteLLMComponent(CustomComponent): max_retries=max_retries, ) return LLM -# litellm.drop_params=True \ No newline at end of file From 08c2542416dd51d4bbf0c40d3fac1915a5f070d7 Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 15:53:21 +0000 Subject: [PATCH 5/6] Rename LiteLLMComponent to ChatLiteLLMComponent --- src/backend/langflow/components/llms/ChatLiteLLM.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/langflow/components/llms/ChatLiteLLM.py b/src/backend/langflow/components/llms/ChatLiteLLM.py index 01b831bae..788b552ce 100644 --- a/src/backend/langflow/components/llms/ChatLiteLLM.py +++ b/src/backend/langflow/components/llms/ChatLiteLLM.py @@ -6,7 +6,7 @@ from langchain_community.chat_models.litellm import ChatLiteLLM, ChatLiteLLMExce import os -class LiteLLMComponent(CustomComponent): +class ChatLiteLLMComponent(CustomComponent): display_name = "ChatLiteLLM" description = "`LiteLLM` collection of large language models." documentation = "https://python.langchain.com/docs/integrations/chat/litellm" From 534cf7c1a66867929ad48af25f688efefe07ed0d Mon Sep 17 00:00:00 2001 From: Ricardo Henriques Date: Sun, 18 Feb 2024 18:47:50 +0000 Subject: [PATCH 6/6] Changes some parameters for mypy linting compatibility --- .../langflow/components/llms/ChatLiteLLM.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/backend/langflow/components/llms/ChatLiteLLM.py b/src/backend/langflow/components/llms/ChatLiteLLM.py index 788b552ce..59507733f 100644 --- a/src/backend/langflow/components/llms/ChatLiteLLM.py +++ b/src/backend/langflow/components/llms/ChatLiteLLM.py @@ -102,20 +102,20 @@ class ChatLiteLLMComponent(CustomComponent): model_kwargs: Optional[Dict[str, Any]] = {}, top_p: Optional[float] = None, top_k: Optional[int] = None, - n: Optional[int] = 1, - max_tokens: Optional[int] = 256, - max_retries: Optional[int] = 6, - verbose: Optional[bool] = False, + n: int = 1, + max_tokens: int = 256, + max_retries: int = 6, + verbose: bool = False, ) -> Union[BaseLanguageModel, Callable]: try: import litellm + litellm.drop_params=True + litellm.set_verbose=verbose except ImportError: raise ChatLiteLLMException( "Could not import litellm python package. " "Please install it with `pip install litellm`" ) - litellm.drop_params=True - litellm.set_verbose=verbose if api_key: if "perplexity" in model: os.environ["PERPLEXITYAI_API_KEY"] = api_key @@ -124,9 +124,10 @@ class ChatLiteLLMComponent(CustomComponent): LLM = ChatLiteLLM( model=model, + client=None, streaming=streaming, temperature=temperature, - model_kwargs=model_kwargs, + model_kwargs=model_kwargs if model_kwargs is not None else {}, top_p=top_p, top_k=top_k, n=n,