diff --git a/.gitattributes b/.gitattributes index d6e351bc3..379b21be8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -11,12 +11,12 @@ *.ts text *.tsx text *.md text -*.mdx text working-tree-encoding = UTF-8 +*.mdx text *.yml text *.yaml text *.xml text *.csv text -*.json text working-tree-encoding = UTF-8 +*.json text *.sh text *.Dockerfile text Dockerfile text diff --git a/src/backend/base/langflow/api/v1/files.py b/src/backend/base/langflow/api/v1/files.py index f6293686f..a72ee7711 100644 --- a/src/backend/base/langflow/api/v1/files.py +++ b/src/backend/base/langflow/api/v1/files.py @@ -110,7 +110,7 @@ async def download_profile_picture( extension = file_name.split(".")[-1] config_dir = get_storage_service().settings_service.settings.config_dir config_path = Path(config_dir) - folder_path = config_path / 'profile_pictures' / folder_name + folder_path = config_path / "profile_pictures" / folder_name content_type = build_content_type_from_extension(extension) file_content = await storage_service.get_file(flow_id=folder_path, file_name=file_name) return StreamingResponse(BytesIO(file_content), media_type=content_type) @@ -140,7 +140,6 @@ async def list_profile_pictures(storage_service: StorageService = Depends(get_st raise HTTPException(status_code=500, detail=str(e)) - @router.get("/list/{flow_id}") async def list_files( flow_id: UUID = Depends(get_flow_id), storage_service: StorageService = Depends(get_storage_service) diff --git a/src/backend/base/langflow/base/vectorstores/utils.py b/src/backend/base/langflow/base/vectorstores/utils.py index e8d93ff37..739181600 100644 --- a/src/backend/base/langflow/base/vectorstores/utils.py +++ b/src/backend/base/langflow/base/vectorstores/utils.py @@ -15,7 +15,7 @@ def chroma_collection_to_records(collection_dict: dict): for i, doc in enumerate(collection_dict["documents"]): record_dict = { "id": collection_dict["ids"][i], - "document": doc, + "text": doc, } if "metadatas" in collection_dict: for key, value in collection_dict["metadatas"][i].items(): diff --git a/src/backend/base/langflow/components/retrievers/SelfQueryRetriever.py b/src/backend/base/langflow/components/retrievers/SelfQueryRetriever.py index a46bf7bd8..3e6d6f696 100644 --- a/src/backend/base/langflow/components/retrievers/SelfQueryRetriever.py +++ b/src/backend/base/langflow/components/retrievers/SelfQueryRetriever.py @@ -4,7 +4,7 @@ from langchain.retrievers.self_query.base import SelfQueryRetriever from langchain_core.vectorstores import VectorStore from langflow.custom import CustomComponent -from langflow.field_typing import BaseLanguageModel +from langflow.field_typing import BaseLanguageModel, Text from langflow.schema import Record from langflow.schema.message import Message @@ -14,25 +14,54 @@ class SelfQueryRetrieverComponent(CustomComponent): description: str = "Retriever that uses a vector store and an LLM to generate the vector store queries." icon = "LangChain" + def build_config(self): + return { + "query": { + "display_name": "Query", + "input_types": ["Message", "Text"], + "info": "Query to be passed as input.", + }, + "vectorstore": { + "display_name": "Vector Store", + "info": "Vector Store to be passed as input.", + }, + "attribute_infos": { + "display_name": "Metadata Field Info", + "info": "Metadata Field Info to be passed as input.", + }, + "document_content_description": { + "display_name": "Document Content Description", + "info": "Document Content Description to be passed as input.", + }, + "llm": { + "display_name": "LLM", + "info": "LLM to be passed as input.", + }, + } + def build( self, query: Message, vectorstore: VectorStore, - metadata_field_info: list[AttributeInfo], - document_content_description: str, + attribute_infos: list[Record], + document_content_description: Text, llm: BaseLanguageModel, ) -> Record: - metadata_field_info = [i[0] for i in metadata_field_info] - + metadata_field_infos = [AttributeInfo(**record.data) for record in attribute_infos] self_query_retriever = SelfQueryRetriever.from_llm( - llm, - vectorstore, - document_content_description, - metadata_field_info, + llm=llm, + vectorstore=vectorstore, + document_contents=document_content_description, + metadata_field_info=metadata_field_infos, enable_limit=True, ) - input_text = query.text + if isinstance(query, Message): + input_text = query.text + elif isinstance(query, str): + input_text = query + else: + raise ValueError(f"Query type {type(query)} not supported.") documents = self_query_retriever.invoke(input=input_text) records = [Record.from_document(document) for document in documents] self.status = records diff --git a/src/backend/base/langflow/components/vectorstores/Chroma.py b/src/backend/base/langflow/components/vectorstores/Chroma.py index b7cfad7d4..6001b119c 100644 --- a/src/backend/base/langflow/components/vectorstores/Chroma.py +++ b/src/backend/base/langflow/components/vectorstores/Chroma.py @@ -1,3 +1,4 @@ +from copy import deepcopy from typing import List, Optional, Union import chromadb @@ -6,6 +7,7 @@ from langchain_chroma import Chroma from langchain_core.embeddings import Embeddings from langchain_core.retrievers import BaseRetriever from langchain_core.vectorstores import VectorStore + from langflow.base.vectorstores.utils import chroma_collection_to_records from langflow.custom import CustomComponent from langflow.schema import Record @@ -48,6 +50,11 @@ class ChromaComponent(CustomComponent): "display_name": "Server SSL Enabled", "advanced": True, }, + "allow_duplicates": { + "display_name": "Allow Duplicates", + "advanced": True, + "info": "If false, will not add documents that are already in the Vector Store.", + }, } def build( @@ -61,6 +68,7 @@ class ChromaComponent(CustomComponent): chroma_server_host: Optional[str] = None, chroma_server_http_port: Optional[int] = None, chroma_server_grpc_port: Optional[int] = None, + allow_duplicates: bool = False, ) -> Union[VectorStore, BaseRetriever]: """ Builds the Vector Store or BaseRetriever object. @@ -75,6 +83,7 @@ class ChromaComponent(CustomComponent): - chroma_server_host (Optional[str]): The host for the Chroma server. - chroma_server_http_port (Optional[int]): The HTTP port for the Chroma server. - chroma_server_grpc_port (Optional[int]): The gRPC port for the Chroma server. + - allow_duplicates (bool): Whether to allow duplicates in the Vector Store. Returns: - Union[VectorStore, BaseRetriever]: The Vector Store or BaseRetriever object. @@ -93,35 +102,34 @@ class ChromaComponent(CustomComponent): ) client = chromadb.HttpClient(settings=chroma_settings) - # If documents, then we need to create a Chroma instance using .from_documents - # Check index_directory and expand it if it is a relative path if index_directory is not None: index_directory = self.resolve_path(index_directory) + chroma = Chroma( + persist_directory=index_directory, + client=client, + embedding_function=embedding, + collection_name=collection_name, + ) + if allow_duplicates: + stored_records = [] + else: + stored_records = chroma_collection_to_records(chroma.get()) + _stored_documents_without_id = [] + for record in deepcopy(stored_records): + del record.id + _stored_documents_without_id.append(record) documents = [] for _input in inputs or []: if isinstance(_input, Record): - documents.append(_input.to_lc_document()) + if _input not in _stored_documents_without_id: + documents.append(_input.to_lc_document()) else: - documents.append(_input) - if documents is not None and embedding is not None: - if len(documents) == 0: - raise ValueError("If documents are provided, there must be at least one document.") - chroma = Chroma.from_documents( - documents=documents, # type: ignore - persist_directory=index_directory, - collection_name=collection_name, - embedding=embedding, - client=client, - ) - else: - chroma = Chroma( - persist_directory=index_directory, - client=client, - embedding_function=embedding, - ) + raise ValueError("Inputs must be a Record objects.") - store = chroma.get() - self.status = chroma_collection_to_records(store) + if documents and embedding is not None: + chroma.add_documents(documents) + + self.status = stored_records return chroma diff --git a/src/backend/base/langflow/helpers/folders.py b/src/backend/base/langflow/helpers/folders.py index fa6f27fcc..c3d7567b5 100644 --- a/src/backend/base/langflow/helpers/folders.py +++ b/src/backend/base/langflow/helpers/folders.py @@ -20,4 +20,4 @@ def generate_unique_folder_name(folder_name, user_id, session): # If a folder with the name already exists, append (n) to the name and increment n folder_name = f"{original_name} ({n})" - n += 1 \ No newline at end of file + n += 1 diff --git a/src/frontend/.prettierignore b/src/frontend/.prettierignore index 07ed7069a..a007feab0 100644 --- a/src/frontend/.prettierignore +++ b/src/frontend/.prettierignore @@ -1 +1 @@ -build/* \ No newline at end of file +build/* diff --git a/src/frontend/package.json b/src/frontend/package.json index 63cbf6503..ab53c6e0b 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -82,7 +82,7 @@ "start": "vite", "build": "vite build", "serve": "vite preview", - "format": "npx prettier --write \"{docs,src}/**/*.{js,jsx,ts,tsx,json,md}\" --ignore-path .prettierignore", + "format": "npx prettier --write \"{tests,src}/**/*.{js,jsx,ts,tsx,json,md}\" --ignore-path .prettierignore", "type-check": "tsc --noEmit --pretty --project tsconfig.json && vite" }, "simple-git-hooks": { diff --git a/src/frontend/src/CustomNodes/GenericNode/index.tsx b/src/frontend/src/CustomNodes/GenericNode/index.tsx index f66e1e638..b9e939c5d 100644 --- a/src/frontend/src/CustomNodes/GenericNode/index.tsx +++ b/src/frontend/src/CustomNodes/GenericNode/index.tsx @@ -13,7 +13,7 @@ import { TOOLTIP_OUTDATED_NODE, } from "../../constants/constants"; import { BuildStatus } from "../../constants/enums"; -import { getSpecificClassFromBuildStatus } from "../helpers/get-class-from-build-status"; +import { postCustomComponent } from "../../controllers/API"; import NodeToolbarComponent from "../../pages/FlowPage/components/nodeToolbarComponent"; import useAlertStore from "../../stores/alertStore"; import { useDarkStore } from "../../stores/darkStore"; @@ -25,6 +25,8 @@ import { NodeDataType } from "../../types/flow"; import { handleKeyDown, scapedJSONStringfy } from "../../utils/reactflowUtils"; import { nodeColors, nodeIconsLucide } from "../../utils/styleUtils"; import { classNames, cn } from "../../utils/utils"; +import { countHandlesFn } from "../helpers/count-handles"; +import { getSpecificClassFromBuildStatus } from "../helpers/get-class-from-build-status"; import useCheckCodeValidity from "../hooks/use-check-code-validity"; import useIconNodeRender from "../hooks/use-icon-render"; import useIconStatus from "../hooks/use-icons-status"; @@ -34,9 +36,7 @@ import useValidationStatusString from "../hooks/use-validation-status-string"; import getFieldTitle from "../utils/get-field-title"; import sortFields from "../utils/sort-fields"; import ParameterComponent from "./components/parameterComponent"; -import { postCustomComponent } from "../../controllers/API"; import { useShortcutsStore } from "../../stores/shortcuts"; -import { countHandlesFn } from "../helpers/count-handles"; export default function GenericNode({ data, diff --git a/src/frontend/src/components/chatComponent/index.tsx b/src/frontend/src/components/chatComponent/index.tsx index 7fad5a882..5bcaade0a 100644 --- a/src/frontend/src/components/chatComponent/index.tsx +++ b/src/frontend/src/components/chatComponent/index.tsx @@ -1,6 +1,6 @@ import { Transition } from "@headlessui/react"; +import { useMemo, useRef, useState } from "react"; import { useHotkeys } from "react-hotkeys-hook"; -import { useEffect, useMemo, useRef, useState } from "react"; import IOModal from "../../modals/IOModal"; import ApiModal from "../../modals/apiModal/views"; import ShareModal from "../../modals/shareModal"; diff --git a/src/frontend/src/components/codeTabsComponent/index.tsx b/src/frontend/src/components/codeTabsComponent/index.tsx index 3b42608d1..0611de7cc 100644 --- a/src/frontend/src/components/codeTabsComponent/index.tsx +++ b/src/frontend/src/components/codeTabsComponent/index.tsx @@ -26,7 +26,6 @@ import { TabsTrigger, } from "../../components/ui/tabs"; import { LANGFLOW_SUPPORTED_TYPES } from "../../constants/constants"; -import ExportModal from "../../modals/exportModal"; import { Case } from "../../shared/components/caseComponent"; import { useDarkStore } from "../../stores/darkStore"; import useFlowStore from "../../stores/flowStore"; @@ -43,9 +42,9 @@ import IconComponent from "../genericIconComponent"; import InputComponent from "../inputComponent"; import KeypairListComponent from "../keypairListComponent"; import ShadTooltip from "../shadTooltipComponent"; +import { Button } from "../ui/button"; import { Label } from "../ui/label"; import { Switch } from "../ui/switch"; -import { Button } from "../ui/button"; export default function CodeTabsComponent({ flow, diff --git a/src/frontend/src/components/headerComponent/components/menuBar/index.tsx b/src/frontend/src/components/headerComponent/components/menuBar/index.tsx index 6405e258a..4965d37a6 100644 --- a/src/frontend/src/components/headerComponent/components/menuBar/index.tsx +++ b/src/frontend/src/components/headerComponent/components/menuBar/index.tsx @@ -16,11 +16,11 @@ import FlowSettingsModal from "../../../../modals/flowSettingsModal"; import useAlertStore from "../../../../stores/alertStore"; import useFlowStore from "../../../../stores/flowStore"; import useFlowsManagerStore from "../../../../stores/flowsManagerStore"; +import { useShortcutsStore } from "../../../../stores/shortcuts"; import { cn } from "../../../../utils/utils"; import IconComponent from "../../../genericIconComponent"; import ShadTooltip from "../../../shadTooltipComponent"; import { Button } from "../../../ui/button"; -import { useShortcutsStore } from "../../../../stores/shortcuts"; export const MenuBar = ({}: {}): JSX.Element => { const shortcuts = useShortcutsStore((state) => state.shortcuts); @@ -115,7 +115,7 @@ export const MenuBar = ({}: {}): JSX.Element => { title: UPLOAD_ERROR_ALERT, list: [error], }); - } + }, ); }} > @@ -201,7 +201,7 @@ export const MenuBar = ({}: {}): JSX.Element => { name={isBuilding || saveLoading ? "Loader2" : "CheckCircle2"} className={cn( "h-4 w-4", - isBuilding || saveLoading ? "animate-spin" : "animate-wiggle" + isBuilding || saveLoading ? "animate-spin" : "animate-wiggle", )} /> {printByBuildStatus()} diff --git a/src/frontend/src/components/headerComponent/index.tsx b/src/frontend/src/components/headerComponent/index.tsx index ba8ee5bf2..11a774b84 100644 --- a/src/frontend/src/components/headerComponent/index.tsx +++ b/src/frontend/src/components/headerComponent/index.tsx @@ -1,9 +1,9 @@ import { useContext } from "react"; -import profileCircle from "../../assets/profile-circle.png"; import { FaDiscord, FaGithub } from "react-icons/fa"; import { RiTwitterXFill } from "react-icons/ri"; import { Link, useLocation, useNavigate, useParams } from "react-router-dom"; import AlertDropdown from "../../alerts/alertDropDown"; +import profileCircle from "../../assets/profile-circle.png"; import { BACKEND_URL, BASE_URL_API, @@ -18,7 +18,6 @@ import useFlowStore from "../../stores/flowStore"; import useFlowsManagerStore from "../../stores/flowsManagerStore"; import { useLocationStore } from "../../stores/locationStore"; import { useStoreStore } from "../../stores/storeStore"; -import { gradients } from "../../utils/styleUtils"; import IconComponent from "../genericIconComponent"; import { Button } from "../ui/button"; import { @@ -202,8 +201,9 @@ export default function Header(): JSX.Element { `${BACKEND_URL.slice( 0, BACKEND_URL.length - 1, - )}${BASE_URL_API}files/profile_pictures/${userData?.profile_image ?? "Space/046-rocket.png"}` ?? - profileCircle + )}${BASE_URL_API}files/profile_pictures/${ + userData?.profile_image ?? "Space/046-rocket.png" + }` ?? profileCircle } className="h-7 w-7 shrink-0 focus-visible:outline-0" /> @@ -219,8 +219,9 @@ export default function Header(): JSX.Element { `${BACKEND_URL.slice( 0, BACKEND_URL.length - 1, - )}${BASE_URL_API}files/profile_pictures/${userData?.profile_image}` ?? - profileCircle + )}${BASE_URL_API}files/profile_pictures/${ + userData?.profile_image + }` ?? profileCircle } className="h-5 w-5 focus-visible:outline-0 " /> diff --git a/src/frontend/src/components/tableComponent/components/tableAutoCellRender/index.tsx b/src/frontend/src/components/tableComponent/components/tableAutoCellRender/index.tsx index 4a6020f78..903068c84 100644 --- a/src/frontend/src/components/tableComponent/components/tableAutoCellRender/index.tsx +++ b/src/frontend/src/components/tableComponent/components/tableAutoCellRender/index.tsx @@ -1,6 +1,5 @@ import { CustomCellRendererProps } from "ag-grid-react"; import { cn, isTimeStampString } from "../../../../utils/utils"; -import ArrayReader from "../../../arrayReaderComponent"; import DateReader from "../../../dateReaderComponent"; import NumberReader from "../../../numberReader"; import ObjectRender from "../../../objectRender"; diff --git a/src/frontend/src/controllers/API/api.tsx b/src/frontend/src/controllers/API/api.tsx index 4070b784f..b8035e74e 100644 --- a/src/frontend/src/controllers/API/api.tsx +++ b/src/frontend/src/controllers/API/api.tsx @@ -1,4 +1,4 @@ -import axios, { Axios, AxiosError, AxiosInstance } from "axios"; +import axios, { AxiosError, AxiosInstance } from "axios"; import { useContext, useEffect } from "react"; import { Cookies } from "react-cookie"; import { renewAccessToken } from "."; diff --git a/src/frontend/src/icons/LangChain/LangChainIcon.jsx b/src/frontend/src/icons/LangChain/LangChainIcon.jsx index c68bd3f4f..aa5e98bec 100644 --- a/src/frontend/src/icons/LangChain/LangChainIcon.jsx +++ b/src/frontend/src/icons/LangChain/LangChainIcon.jsx @@ -1,10 +1,24 @@ const SvgLangChainIcon = (props) => ( - - - - - - - + + + + + ); export default SvgLangChainIcon; diff --git a/src/frontend/src/index.tsx b/src/frontend/src/index.tsx index 9c828e086..006d93ff2 100644 --- a/src/frontend/src/index.tsx +++ b/src/frontend/src/index.tsx @@ -8,7 +8,6 @@ import "./style/index.css"; // @ts-ignore import "./style/applies.css"; // @ts-ignore -import { StrictMode } from "react"; import "./style/classes.css"; const root = ReactDOM.createRoot( diff --git a/src/frontend/src/modals/IOModal/components/SessionView/index.tsx b/src/frontend/src/modals/IOModal/components/SessionView/index.tsx index b20ed6c57..edabbe252 100644 --- a/src/frontend/src/modals/IOModal/components/SessionView/index.tsx +++ b/src/frontend/src/modals/IOModal/components/SessionView/index.tsx @@ -1,20 +1,10 @@ -import { - CellEditRequestEvent, - ColDef, - ColGroupDef, - SelectionChangedEvent, -} from "ag-grid-community"; +import { CellEditRequestEvent, SelectionChangedEvent } from "ag-grid-community"; import { useState } from "react"; import TableComponent from "../../../../components/tableComponent"; -import { Card, CardContent } from "../../../../components/ui/card"; +import useRemoveMessages from "../../../../pages/SettingsPage/pages/messagesPage/hooks/use-remove-messages"; +import useUpdateMessage from "../../../../pages/SettingsPage/pages/messagesPage/hooks/use-updateMessage"; import useAlertStore from "../../../../stores/alertStore"; import { useMessagesStore } from "../../../../stores/messagesStore"; -import useUpdateMessage from "../../../../pages/SettingsPage/pages/messagesPage/hooks/use-updateMessage"; -import useRemoveMessages from "../../../../pages/SettingsPage/pages/messagesPage/hooks/use-remove-messages"; -import HeaderMessagesComponent from "../../../../pages/SettingsPage/pages/messagesPage/components/headerMessages"; -import { Button } from "../../../../components/ui/button"; -import ForwardedIconComponent from "../../../../components/genericIconComponent"; -import { cn } from "../../../../utils/utils"; export default function SessionView({ rows }: { rows: Array }) { const columns = useMessagesStore((state) => state.columns); diff --git a/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/uploadFileButton/index.tsx b/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/uploadFileButton/index.tsx index e675794e0..17ca897ea 100644 --- a/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/uploadFileButton/index.tsx +++ b/src/frontend/src/modals/IOModal/components/chatView/chatInput/components/uploadFileButton/index.tsx @@ -18,7 +18,9 @@ const UploadFileButton = ({ />