From 99ef7c728dc997aa578f70ba9ddb5346680c403b Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 14 Jul 2023 14:05:57 -0300 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=A8=20refactor(conftest.py):=20ref?= =?UTF-8?q?ormat=20code=20for=20better=20readability=20and=20maintainabili?= =?UTF-8?q?ty=20=E2=9C=A8=20feat(conftest.py):=20add=20MyCustomChain=20cla?= =?UTF-8?q?ss=20as=20an=20example=20of=20a=20custom=20chain=20=E2=9C=A8=20?= =?UTF-8?q?feat(conftest.py):=20add=20CustomChain=20class=20as=20a=20custo?= =?UTF-8?q?m=20component=20for=20building=20a=20document=20=E2=9C=A8=20fea?= =?UTF-8?q?t(conftest.py):=20add=20CSVLoaderComponent=20class=20as=20a=20c?= =?UTF-8?q?ustom=20component=20for=20loading=20CSV=20files=20and=20convert?= =?UTF-8?q?ing=20rows=20to=20documents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 210 ++++++++++++++++++++++++++-------------------- 1 file changed, 120 insertions(+), 90 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ca0bb1dc0..328a168ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,116 +122,146 @@ def client_fixture(session: Session): # def custom_chain(): return '''from __future__ import annotations - from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional - from pydantic import Extra +from pydantic import Extra - from langchain.schema import BaseLanguageModel, Document - from langchain.callbacks.manager import ( - AsyncCallbackManagerForChainRun, - CallbackManagerForChainRun, - ) - from langchain.chains.base import Chain - from langchain.prompts import StringPromptTemplate - from langflow.interface.custom.base import CustomComponent +from langchain.schema import BaseLanguageModel, Document +from langchain.callbacks.manager import ( + AsyncCallbackManagerForChainRun, + CallbackManagerForChainRun, +) +from langchain.chains.base import Chain +from langchain.prompts import StringPromptTemplate +from langflow.interface.custom.base import CustomComponent - class MyCustomChain(Chain): +class MyCustomChain(Chain): + """ + An example of a custom chain. + """ + + prompt: StringPromptTemplate + """Prompt object to use.""" + llm: BaseLanguageModel + output_key: str = "text" #: :meta private: + + class Config: + """Configuration for this pydantic object.""" + + extra = Extra.forbid + arbitrary_types_allowed = True + + @property + def input_keys(self) -> List[str]: + """Will be whatever keys the prompt expects. + + :meta private: """ - An example of a custom chain. + return self.prompt.input_variables + + @property + def output_keys(self) -> List[str]: + """Will always return text key. + + :meta private: """ + return [self.output_key] - prompt: StringPromptTemplate - """Prompt object to use.""" - llm: BaseLanguageModel - output_key: str = "text" #: :meta private: + def _call( + self, + inputs: Dict[str, Any], + run_manager: Optional[CallbackManagerForChainRun] = None, + ) -> Dict[str, str]: + # Your custom chain logic goes here + # This is just an example that mimics LLMChain + prompt_value = self.prompt.format_prompt(**inputs) - class Config: - """Configuration for this pydantic object.""" + # Whenever you call a language model, or another chain, you should pass + # a callback manager to it. This allows the inner run to be tracked by + # any callbacks that are registered on the outer run. + # You can always obtain a callback manager for this by calling + # `run_manager.get_child()` as shown below. + response = self.llm.generate_prompt( + [prompt_value], + callbacks=run_manager.get_child() if run_manager else None, + ) - extra = Extra.forbid - arbitrary_types_allowed = True + # If you want to log something about this run, you can do so by calling + # methods on the `run_manager`, as shown below. This will trigger any + # callbacks that are registered for that event. + if run_manager: + run_manager.on_text("Log something about this run") - @property - def input_keys(self) -> List[str]: - """Will be whatever keys the prompt expects. + return {self.output_key: response.generations[0][0].text} - :meta private: - """ - return self.prompt.input_variables + async def _acall( + self, + inputs: Dict[str, Any], + run_manager: Optional[AsyncCallbackManagerForChainRun] = None, + ) -> Dict[str, str]: + # Your custom chain logic goes here + # This is just an example that mimics LLMChain + prompt_value = self.prompt.format_prompt(**inputs) - @property - def output_keys(self) -> List[str]: - """Will always return text key. + # Whenever you call a language model, or another chain, you should pass + # a callback manager to it. This allows the inner run to be tracked by + # any callbacks that are registered on the outer run. + # You can always obtain a callback manager for this by calling + # `run_manager.get_child()` as shown below. + response = await self.llm.agenerate_prompt( + [prompt_value], + callbacks=run_manager.get_child() if run_manager else None, + ) - :meta private: - """ - return [self.output_key] + # If you want to log something about this run, you can do so by calling + # methods on the `run_manager`, as shown below. This will trigger any + # callbacks that are registered for that event. + if run_manager: + await run_manager.on_text("Log something about this run") - def _call( - self, - inputs: Dict[str, Any], - run_manager: Optional[CallbackManagerForChainRun] = None, - ) -> Dict[str, str]: - # Your custom chain logic goes here - # This is just an example that mimics LLMChain - prompt_value = self.prompt.format_prompt(**inputs) + return {self.output_key: response.generations[0][0].text} - # Whenever you call a language model, or another chain, you should pass - # a callback manager to it. This allows the inner run to be tracked by - # any callbacks that are registered on the outer run. - # You can always obtain a callback manager for this by calling - # `run_manager.get_child()` as shown below. - response = self.llm.generate_prompt( - [prompt_value], - callbacks=run_manager.get_child() if run_manager else None, - ) + @property + def _chain_type(self) -> str: + return "my_custom_chain" - # If you want to log something about this run, you can do so by calling - # methods on the `run_manager`, as shown below. This will trigger any - # callbacks that are registered for that event. - if run_manager: - run_manager.on_text("Log something about this run") +class CustomChain(CustomComponent): + display_name: str = "Custom Chain" + field_config = { + "prompt": {"field_type": "prompt"}, + "llm": {"field_type": "BaseLanguageModel"}, + } - return {self.output_key: response.generations[0][0].text} + def build(self, prompt, llm, input: str) -> Document: + chain = MyCustomChain(prompt=prompt, llm=llm) + return chain(input)''' - async def _acall( - self, - inputs: Dict[str, Any], - run_manager: Optional[AsyncCallbackManagerForChainRun] = None, - ) -> Dict[str, str]: - # Your custom chain logic goes here - # This is just an example that mimics LLMChain - prompt_value = self.prompt.format_prompt(**inputs) - # Whenever you call a language model, or another chain, you should pass - # a callback manager to it. This allows the inner run to be tracked by - # any callbacks that are registered on the outer run. - # You can always obtain a callback manager for this by calling - # `run_manager.get_child()` as shown below. - response = await self.llm.agenerate_prompt( - [prompt_value], - callbacks=run_manager.get_child() if run_manager else None, - ) +@pytest.fixture +def data_processing(): + return """import pandas as pd +from langchain.schema import Document +from langflow.interface.custom.base import CustomComponent - # If you want to log something about this run, you can do so by calling - # methods on the `run_manager`, as shown below. This will trigger any - # callbacks that are registered for that event. - if run_manager: - await run_manager.on_text("Log something about this run") +class CSVLoaderComponent(CustomComponent): + display_name: str = "CSV Loader" + field_config = { + "filename": {"field_type": "str", "required": True}, + "column_name": {"field_type": "str", "required": True}, + } - return {self.output_key: response.generations[0][0].text} + def build(self, filename: str, column_name: str) -> List[Document]: + # Load the CSV file + df = pd.read_csv(filename) - @property - def _chain_type(self) -> str: - return "my_custom_chain" + # Verify the column exists + if column_name not in df.columns: + raise ValueError(f"Column '{column_name}' not found in the CSV file") - class CustomChain(CustomComponent): - display_name: str = "Custom Chain" - field_config = { - "prompt": {"field_type": "prompt"}, - "llm": {"field_type": "BaseLanguageModel"}, - } + # Convert each row of the specified column to a document object + documents = [] + for content in df[column_name]: + metadata = {"filename": filename} + documents.append(Document(page_content=str(content), metadata=metadata)) - def build(self, prompt, llm, input: str) -> Document: - chain = MyCustomChain(prompt=prompt, llm=llm) - return chain(input)''' + return documents""" From 49029d6cdab54a841618e34460628af71046f8da Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 14 Jul 2023 14:13:27 -0300 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=94=A7=20chore(base.py):=20refactor?= =?UTF-8?q?=20TemplateFieldCreator=20class=20to=20improve=20code=20readabi?= =?UTF-8?q?lity=20and=20maintainability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/backend/langflow/template/field/base.py | 36 ++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/backend/langflow/template/field/base.py b/src/backend/langflow/template/field/base.py index dcdb3ea92..31c68d094 100644 --- a/src/backend/langflow/template/field/base.py +++ b/src/backend/langflow/template/field/base.py @@ -6,24 +6,58 @@ from pydantic import BaseModel class TemplateFieldCreator(BaseModel, ABC): field_type: str = "str" + """The type of field this is. Default is a string.""" + required: bool = False + """Specifies if the field is required. Defaults to False.""" + placeholder: str = "" + """A placeholder string for the field. Default is an empty string.""" + is_list: bool = False + """Defines if the field is a list. Default is False.""" + show: bool = True + """Should the field be shown. Defaults to True.""" + multiline: bool = False + """Defines if the field will allow the user to open a text editor. Default is False.""" + value: Any = None + """The value of the field. Default is None.""" + suffixes: list[str] = [] - fileTypes: list[str] = [] + """List of suffixes for a file field. Default is an empty list.""" + file_types: list[str] = [] + """List of file types associated with the field. Default is an empty list. (duplicate)""" + file_path: Union[str, None] = None + """The file path of the field if it is a file. Defaults to None.""" + password: bool = False + """Specifies if the field is a password. Defaults to False.""" + options: list[str] = [] + """List of options for the field. Only used when is_list=True. Default is an empty list.""" + name: str = "" + """Name of the field. Default is an empty string.""" + display_name: Optional[str] = None + """Display name of the field. Defaults to None.""" + advanced: bool = False + """Specifies if the field will an advanced parameter (hidden). Defaults to False.""" + input_types: list[str] = [] + """List of input types for the handle when the field has more than one type. Default is an empty list.""" + dynamic: bool = False + """Specifies if the field is dynamic. Defaults to False.""" + info: Optional[str] = "" + """Additional information about the field to be shown in the tooltip. Defaults to an empty string.""" def to_dict(self): result = self.dict() From 1863d463d0deab90f8418d0f84be31ba21d7e13c Mon Sep 17 00:00:00 2001 From: Gabriel Luiz Freitas Almeida Date: Fri, 14 Jul 2023 14:20:09 -0300 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=94=A8=20refactor(conftest.py):=20cha?= =?UTF-8?q?nge=20return=20type=20of=20`build`=20method=20in=20`CSVLoaderCo?= =?UTF-8?q?mponent`=20from=20List[Document]=20to=20Document=20to=20match?= =?UTF-8?q?=20the=20actual=20return=20type=20=F0=9F=94=A7=20chore(conftest?= =?UTF-8?q?.py):=20add=20new=20fixture=20`filter=5Fdocs`=20for=20testing?= =?UTF-8?q?=20`DocumentFilterByLengthComponent`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/conftest.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 328a168ad..79704e3b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -250,7 +250,7 @@ class CSVLoaderComponent(CustomComponent): "column_name": {"field_type": "str", "required": True}, } - def build(self, filename: str, column_name: str) -> List[Document]: + def build(self, filename: str, column_name: str) -> Document: # Load the CSV file df = pd.read_csv(filename) @@ -265,3 +265,23 @@ class CSVLoaderComponent(CustomComponent): documents.append(Document(page_content=str(content), metadata=metadata)) return documents""" + + +@pytest.fixture +def filter_docs(): + return """from langchain.schema import Document +from langflow.interface.custom.base import CustomComponent +from typing import List + +class DocumentFilterByLengthComponent(CustomComponent): + display_name: str = "Document Filter By Length" + field_config = { + "documents": {"field_type": "Document", "required": True}, + "max_length": {"field_type": "int", "required": True}, + } + + def build(self, documents: List[Document], max_length: int) -> List[Document]: + # Filter the documents by length + filtered_documents = [doc for doc in documents if len(doc.page_content) <= max_length] + + return filtered_documents"""