feat: voice mode tts endpoint (#7294)

* 📝 (chat-view-wrapper.tsx): Refactor ChatViewWrapper component to improve code readability and maintainability
📝 (chat-input.tsx): Add functionality to set voice assistant active state when showAudioInput is true
📝 (voice-assistant.tsx): Add functionality to set voice assistant active state and scroll to bottom when closing audio input
📝 (chat-view.tsx): Update ChatView component to consider sidebarOpen and isVoiceAssistantActive states
📝 (voiceStore.ts): Add isVoiceAssistantActive state and setIsVoiceAssistantActive function to voice store
📝 (index.ts, voice.types.ts): Update types to include sidebarOpen prop in chatViewProps and isVoiceAssistantActive state in VoiceStoreType

* url change

* 🔧 (chat-input.tsx): Add new session close voice assistant functionality to chat input component
🔧 (voice-button.tsx): Update voice button to set new session close voice assistant state
🔧 (sidebar-open-view.tsx): Update sidebar open view to set new session close voice assistant state
🔧 (voiceStore.ts, voice.types.ts): Add new session close voice assistant state and setter to voice store and types

* ♻️ (chat-input.tsx): remove unused setNewSessionCloseVoiceAssistant function to clean up code and improve readability

* url

* merge

* 📝 (chat-view-wrapper.tsx): Refactor ChatViewWrapper component to improve code readability and maintainability
📝 (chat-input.tsx): Add functionality to set voice assistant active state when showAudioInput is true
📝 (voice-assistant.tsx): Add functionality to set voice assistant active state and scroll to bottom when closing audio input
📝 (chat-view.tsx): Update ChatView component to consider sidebarOpen and isVoiceAssistantActive states
📝 (voiceStore.ts): Add isVoiceAssistantActive state and setIsVoiceAssistantActive function to voice store
📝 (index.ts, voice.types.ts): Update types to include sidebarOpen prop in chatViewProps and isVoiceAssistantActive state in VoiceStoreType

* new endpoint

* 🐛 (voice_mode.py): remove unnecessary comment and update input parameter in speech creation function
🐛 (use-start-conversation.ts): update WebSocket URL to use flow_tts endpoint and add support for audio language and input audio transcription model in WebSocket session update configuration

* 📝 (voice_mode.py): add "voice" attribute with value "coral" to TTSConfig class for voice mode customization
♻️ (use-start-conversation.ts): refactor code to use "transcription_session.update" type and update session attributes based on audioSettings and audioLanguage variables

* ✨ (voice_mode.py): introduce a new 'voice' attribute with the value "coral" to specify the voice used for text-to-speech conversion in TTSConfig class

* ♻️ (voice_mode.py): remove unused 'voice' variable assignment to improve code cleanliness and readability

* ✨ (voice_mode.py): update voice parameter value to "coral" for better voice quality and clarity in TTS websocket flow.

* ✨ (voice_mode.py): add support for configuring OpenAI voice setting in TTSConfig class
🐛 (voice_mode.py): fix updating OpenAI voice setting in flow_tts_websocket function
📝 (use-start-conversation.ts): update voice settings format for flow_tts endpoint to include voice and provider information

* 🐛 (use-start-conversation.ts): fix issue with immediate startRecording function call by adding a 300ms delay to ensure proper initialization

* refactor

* refactor

* 🔧 (audio-settings-dialog.tsx): add isPlayingRef prop to SettingsVoiceModal component for managing audio playback state
🔧 (voice-assistant.tsx): pass isPlayingRef prop to VoiceAssistant component for controlling audio playback state

* [autofix.ci] apply automated fixes

* fixed tts and 11l

* lint fix

---------

Co-authored-by: cristhianzl <cristhian.lousa@gmail.com>
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

src/frontend/src/controllers/API/queries/messages/use-get-messages-polling.ts

@@ -14,6 +14,7 @@ interface MessagesQueryParams {
   params?: object;
   onSuccess?: (data: MessagesResponse) => void;
   stopPollingOn?: (data: MessagesResponse) => boolean;
+  session_id?: string;
 }
 
 interface MessagesResponse {
@@ -109,6 +110,7 @@ export const useGetMessagesPollingMutation = (
     payload: MessagesQueryParams,
   ): Promise<MessagesResponse> => {
     const requestId = payload.id || "default";
+    const sessionId = payload.session_id;
 
     if (requestInProgressRef.current[requestId]) {
       return Promise.reject("Request already in progress");
@@ -118,9 +120,11 @@ export const useGetMessagesPollingMutation = (
       requestInProgressRef.current[requestId] = true;
       const { id, mode, excludedFields, params } = payload;
       const config = {};
+
       if (id) {
         config["params"] = { flow_id: id };
       }
+
       if (params) {
         config["params"] = { ...config["params"], ...params };
       }

src/frontend/src/modals/IOModal/components/IOFieldView/components/session-selector.tsx

@@ -11,6 +11,7 @@ import { useUpdateSessionName } from "@/controllers/API/queries/messages/use-ren
 import useFlowsManagerStore from "@/stores/flowsManagerStore";
 import useFlowStore from "@/stores/flowStore";
 import { useUtilityStore } from "@/stores/utilityStore";
+import { useVoiceStore } from "@/stores/voiceStore";
 import { cn } from "@/utils/utils";
 import React, { useEffect, useRef, useState } from "react";
 import { v5 as uuidv5 } from "uuid";
@@ -117,10 +118,15 @@ export default function SessionSelector({
     }
   };
 
+  const setNewSessionCloseVoiceAssistant = useVoiceStore(
+    (state) => state.setNewSessionCloseVoiceAssistant,
+  );
+
   return (
     <div
       data-testid="session-selector"
       onClick={(e) => {
+        setNewSessionCloseVoiceAssistant(true);
         if (isEditing) e.stopPropagation();
         else toggleVisibility();
       }}

src/frontend/src/modals/IOModal/components/chat-view-wrapper.tsx

@@ -91,30 +91,23 @@ export const ChatViewWrapper = ({
           {!playgroundPage && <Separator orientation="vertical" />}
         </div>
       </div>
-      <div
-        className={cn(
-          visibleSession ? "h-[95%]" : "h-full",
-          sidebarOpen
-            ? "pointer-events-none blur-sm lg:pointer-events-auto lg:blur-0"
-            : "",
-        )}
-      >
-        {messagesFetched && (
-          <ChatView
-            focusChat={sessionId}
-            sendMessage={sendMessage}
-            visibleSession={visibleSession}
-            closeChat={
-              !canvasOpen
-                ? undefined
-                : () => {
-                    setOpen(false);
-                  }
-            }
-            playgroundPage={playgroundPage}
-          />
-        )}
-      </div>
+
+      {messagesFetched && (
+        <ChatView
+          focusChat={sessionId}
+          sendMessage={sendMessage}
+          visibleSession={visibleSession}
+          closeChat={
+            !canvasOpen
+              ? undefined
+              : () => {
+                  setOpen(false);
+                }
+          }
+          playgroundPage={playgroundPage}
+          sidebarOpen={sidebarOpen}
+        />
+      )}
     </div>
   );
 };

src/frontend/src/modals/IOModal/components/chatView/chatInput/chat-input.tsx

@@ -3,6 +3,7 @@ import useFileSizeValidator from "@/shared/hooks/use-file-size-validator";
 import useAlertStore from "@/stores/alertStore";
 import useFlowStore from "@/stores/flowStore";
 import { useUtilityStore } from "@/stores/utilityStore";
+import { useVoiceStore } from "@/stores/voiceStore";
 import { AnimatePresence, motion } from "framer-motion";
 import { useEffect, useRef, useState } from "react";
 import ShortUniqueId from "short-unique-id";
@@ -41,6 +42,20 @@ export default function ChatInput({
 
   const [showAudioInput, setShowAudioInput] = useState(false);
 
+  const setIsVoiceAssistantActive = useVoiceStore(
+    (state) => state.setIsVoiceAssistantActive,
+  );
+
+  const newSessionCloseVoiceAssistant = useVoiceStore(
+    (state) => state.newSessionCloseVoiceAssistant,
+  );
+
+  useEffect(() => {
+    if (showAudioInput) {
+      setIsVoiceAssistantActive(true);
+    }
+  }, [showAudioInput]);
+
   useFocusOnUnlock(isBuilding, inputRef);
   useAutoResizeTextArea(chatValue, inputRef);
 
@@ -184,7 +199,7 @@ export default function ChatInput({
 
   return (
     <AnimatePresence mode="wait">
-      {showAudioInput ? (
+      {showAudioInput && !newSessionCloseVoiceAssistant ? (
         <motion.div
           key="voice-assistant"
           initial={{ opacity: 0 }}

src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/components/audio-settings/audio-settings-dialog.tsx

@@ -37,6 +37,7 @@ interface SettingsVoiceModalProps {
   handleClickSaveOpenAIApiKey: (openaiApiKey: string) => void;
   isEditingOpenAIKey: boolean;
   setIsEditingOpenAIKey: (isEditingOpenAIKey: boolean) => void;
+  isPlayingRef: React.MutableRefObject<boolean>;
 }
 
 const SettingsVoiceModal = ({
@@ -50,6 +51,7 @@ const SettingsVoiceModal = ({
   handleClickSaveOpenAIApiKey,
   isEditingOpenAIKey,
   setIsEditingOpenAIKey,
+  isPlayingRef,
 }: SettingsVoiceModalProps) => {
   const popupRef = useRef<HTMLDivElement>(null);
   const [voice, setVoice] = useState<string>("alloy");
@@ -147,6 +149,7 @@ const SettingsVoiceModal = ({
   };
 
   const onOpenChangeDropdownMenu = (open: boolean) => {
+    isPlayingRef.current = false;
     setOpen(open);
     setShowSettingsModal(open, openaiApiKey, elevenLabsApiKey);
   };

src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/components/voice-button.tsx

@@ -1,17 +1,25 @@
 import ForwardedIconComponent from "@/components/common/genericIconComponent";
 import { Button } from "@/components/ui/button";
 import { ICON_STROKE_WIDTH } from "@/constants/constants";
+import { useVoiceStore } from "@/stores/voiceStore";
 
 interface VoiceButtonProps {
   toggleRecording: () => void;
 }
 
 const VoiceButton = ({ toggleRecording }: VoiceButtonProps) => {
+  const setNewSessionCloseVoiceAssistant = useVoiceStore(
+    (state) => state.setNewSessionCloseVoiceAssistant,
+  );
+
   return (
     <>
       <div>
         <Button
-          onClick={toggleRecording}
+          onClick={() => {
+            toggleRecording();
+            setNewSessionCloseVoiceAssistant(false);
+          }}
           className="btn-playground-actions group"
           unstyled
           data-testid="voice-button"

src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/hooks/use-start-conversation.ts

@@ -12,7 +12,8 @@ export const useStartConversation = (
   const currentHost = window.location.hostname;
   const currentPort = window.location.port;
   const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
-  const url = `${protocol}//${currentHost}:${currentPort}/api/v1/voice/ws/flow_as_tool/${flowId}/${currentSessionId}`;
+  const url = `${protocol}//${currentHost}:${currentPort}/api/v1/voice/ws/flow_tts/${flowId}/${currentSessionId?.toString()}`;
+  //const url = `${protocol}//${currentHost}:${currentPort}/api/v1/voice/ws/flow_as_tool/${flowId}/${currentSessionId?.toString()}`;
 
   try {
     if (wsRef.current?.readyState === WebSocket.CONNECTING) {
@@ -26,6 +27,8 @@ export const useStartConversation = (
     const audioSettings = JSON.parse(
       getLocalStorage("lf_audio_settings_playground") || "{}",
     );
+    const audioLanguage =
+      getLocalStorage("lf_audio_language_playground") || "en-US";
 
     wsRef.current = new WebSocket(url);
 
@@ -42,17 +45,20 @@ export const useStartConversation = (
                 : "",
           }),
         );
+
+        // For flow_tts endpoint, we need to use the proper session update format
         if (audioSettings.provider !== "elevenlabs") {
           wsRef.current.send(
             JSON.stringify({
-              type: "session.update",
-              session: {
-                voice: audioSettings.voice,
-              },
+              type: "voice.settings",
+              voice: audioSettings.voice || "echo",
+              provider: audioSettings.provider || "openai",
             }),
           );
         }
-        startRecording();
+        setTimeout(() => {
+          startRecording();
+        }, 300);
       }
     };
 

src/frontend/src/modals/IOModal/components/chatView/chatInput/components/voice-assistant/voice-assistant.tsx

@@ -14,7 +14,6 @@ import { useMessagesStore } from "@/stores/messagesStore";
 import { useUtilityStore } from "@/stores/utilityStore";
 import { useVoiceStore } from "@/stores/voiceStore";
 import { cn } from "@/utils/utils";
-import { AxiosError } from "axios";
 import { useEffect, useMemo, useRef, useState } from "react";
 import IconComponent from "../../../../../../../components/common/genericIconComponent";
 import SettingsVoiceModal from "./components/audio-settings/audio-settings-dialog";
@@ -61,6 +60,9 @@ export function VoiceAssistant({
   const analyserRef = useRef<AnalyserNode | null>(null);
 
   const soundDetected = useVoiceStore((state) => state.soundDetected);
+  const setIsVoiceAssistantActive = useVoiceStore(
+    (state) => state.setIsVoiceAssistantActive,
+  );
   const setSoundDetected = useVoiceStore((state) => state.setSoundDetected);
   const messagesStore = useMessagesStore();
   const setIsBuilding = useFlowStore((state) => state.setIsBuilding);
@@ -86,6 +88,8 @@ export function VoiceAssistant({
   const currentSessionId = useUtilityStore((state) => state.currentSessionId);
   const setErrorData = useAlertStore((state) => state.setErrorData);
   const { data: globalVariables } = useGetGlobalVariables();
+  const currentFlow = useFlowStore((state) => state.currentFlow);
+  const currentFlowId = currentFlow?.id;
 
   const hasOpenAIAPIKey = useMemo(() => {
     return (
@@ -93,13 +97,6 @@ export function VoiceAssistant({
     );
   }, [variables, open, addKey]);
 
-  const hasElevenLabsApiKey = useMemo(() => {
-    return (
-      variables?.find((variable) => variable === "ELEVENLABS_API_KEY")
-        ?.length! > 0
-    );
-  }, [variables, addKey, open]);
-
   const openaiApiKey = useMemo(() => {
     return variables?.find((variable) => variable === "OPENAI_API_KEY");
   }, [variables, addKey]);
@@ -216,7 +213,7 @@ export function VoiceAssistant({
   const handleGetMessagesMutation = () => {
     getMessagesMutation.mutate({
       mode: "union",
-      id: currentSessionId,
+      id: currentFlowId,
     });
   };
 
@@ -291,10 +288,20 @@ export function VoiceAssistant({
     };
   }, []);
 
+  const scrollToBottom = () => {
+    setTimeout(() => {
+      const chatContainer = document.querySelector(".chat-message-div");
+      if (chatContainer) {
+        chatContainer.scrollTop = chatContainer.scrollHeight;
+      }
+    }, 300);
+  };
+
   const handleCloseAudioInput = () => {
-    setIsRecording;
+    setIsRecording(false);
     stopRecording();
     setShowAudioInput(false);
+    scrollToBottom();
   };
 
   const handleSetShowSettingsModal = async (
@@ -430,6 +437,7 @@ export function VoiceAssistant({
               handleClickSaveOpenAIApiKey={handleClickSaveOpenAIApiKey}
               isEditingOpenAIKey={isEditingOpenAIKey}
               setIsEditingOpenAIKey={setIsEditingOpenAIKey}
+              isPlayingRef={isPlayingRef}
             >
               {hasOpenAIAPIKey ? (
                 <>

src/frontend/src/modals/IOModal/components/chatView/components/chat-view.tsx

@@ -39,6 +39,7 @@ export default function ChatView({
   focusChat,
   closeChat,
   playgroundPage,
+  sidebarOpen,
 }: chatViewProps): JSX.Element {
   const flowPool = useFlowStore((state) => state.flowPool);
   const inputs = useFlowStore((state) => state.inputs);
@@ -163,11 +164,19 @@ export default function ChatView({
   };
 
   const flowRunningSkeletonMemo = useMemo(() => <FlowRunningSqueleton />, []);
-  const soundDetected = useVoiceStore((state) => state.soundDetected);
+  const isVoiceAssistantActive = useVoiceStore(
+    (state) => state.isVoiceAssistantActive,
+  );
 
   return (
     <div
-      className="flex h-full w-full flex-col rounded-md"
+      className={cn(
+        "flex h-full w-full flex-col rounded-md",
+        visibleSession ? "h-[95%]" : "h-full",
+        sidebarOpen &&
+          !isVoiceAssistantActive &&
+          "pointer-events-none blur-sm lg:pointer-events-auto lg:blur-0",
+      )}
       onDragOver={dragOver}
       onDragEnter={dragEnter}
       onDragLeave={dragLeave}

src/frontend/src/modals/IOModal/components/sidebar-open-view.tsx

@@ -1,5 +1,6 @@
 import ShadTooltip from "@/components/common/shadTooltipComponent";
 import { Button } from "@/components/ui/button";
+import { useVoiceStore } from "@/stores/voiceStore";
 import IconComponent from "../../../components/common/genericIconComponent";
 import { SidebarOpenViewProps } from "../types/sidebar-open-view";
 import SessionSelector from "./IOFieldView/components/session-selector";
@@ -13,6 +14,10 @@ export const SidebarOpenView = ({
   selectedViewField,
   playgroundPage,
 }: SidebarOpenViewProps) => {
+  const setNewSessionCloseVoiceAssistant = useVoiceStore(
+    (state) => state.setNewSessionCloseVoiceAssistant,
+  );
+
   return (
     <>
       <div className="flex flex-col pl-3">
@@ -34,6 +39,7 @@ export const SidebarOpenView = ({
                   onClick={(_) => {
                     setvisibleSession(undefined);
                     setSelectedViewField(undefined);
+                    setNewSessionCloseVoiceAssistant(true);
                   }}
                 >
                   <IconComponent

src/frontend/src/stores/voiceStore.ts

@@ -37,4 +37,10 @@ export const useVoiceStore = create<VoiceStoreType>((set, get) => ({
   ) => set({ openaiVoices }),
   soundDetected: false,
   setSoundDetected: (soundDetected: boolean) => set({ soundDetected }),
+  isVoiceAssistantActive: false,
+  setIsVoiceAssistantActive: (isVoiceAssistantActive: boolean) =>
+    set({ isVoiceAssistantActive }),
+  newSessionCloseVoiceAssistant: false,
+  setNewSessionCloseVoiceAssistant: (newSessionCloseVoiceAssistant: boolean) =>
+    set({ newSessionCloseVoiceAssistant }),
 }));

src/frontend/src/types/components/index.ts

@@ -792,6 +792,7 @@ export type IOFieldViewProps = {
 export type UndrawCardComponentProps = { flow: FlowType };
 
 export type chatViewProps = {
+  sidebarOpen: boolean;
   sendMessage: ({
     repeat,
     files,

src/frontend/src/types/zustand/voice/voice.types.ts

@@ -31,4 +31,10 @@ export type VoiceStoreType = {
   ) => void;
   soundDetected: boolean;
   setSoundDetected: (soundDetected: boolean) => void;
+  isVoiceAssistantActive: boolean;
+  setIsVoiceAssistantActive: (isVoiceAssistantActive: boolean) => void;
+  newSessionCloseVoiceAssistant: boolean;
+  setNewSessionCloseVoiceAssistant: (
+    newSessionCloseVoiceAssistant: boolean,
+  ) => void;
 };