first pass at turn based conversation

vocode/streaming/models/agent.py

@@ -0,0 +1,181 @@
+from typing import Optional, Union
+from enum import Enum
+
+from pydantic import validator
+
+from vocode.streaming.models.message import BaseMessage
+from .model import TypedModel, BaseModel
+
+FILLER_AUDIO_DEFAULT_SILENCE_THRESHOLD_SECONDS = 0.5
+LLM_AGENT_DEFAULT_TEMPERATURE = 1.0
+LLM_AGENT_DEFAULT_MAX_TOKENS = 256
+LLM_AGENT_DEFAULT_MODEL_NAME = "text-curie-001"
+CHAT_GPT_AGENT_DEFAULT_MODEL_NAME = "gpt-3.5-turbo"
+
+
+class AgentType(str, Enum):
+    BASE = "agent_base"
+    LLM = "agent_llm"
+    CHAT_GPT_ALPHA = "agent_chat_gpt_alpha"
+    CHAT_GPT = "agent_chat_gpt"
+    ECHO = "agent_echo"
+    INFORMATION_RETRIEVAL = "agent_information_retrieval"
+    RESTFUL_USER_IMPLEMENTED = "agent_restful_user_implemented"
+    WEBSOCKET_USER_IMPLEMENTED = "agent_websocket_user_implemented"
+
+
+class FillerAudioConfig(BaseModel):
+    silence_threshold_seconds: float = FILLER_AUDIO_DEFAULT_SILENCE_THRESHOLD_SECONDS
+    use_phrases: bool = True
+    use_typing_noise: bool = False
+
+    @validator("use_typing_noise")
+    def typing_noise_excludes_phrases(cls, v, values):
+        if v and values.get("use_phrases"):
+            values["use_phrases"] = False
+        if not v and not values.get("use_phrases"):
+            raise ValueError("must use either typing noise or phrases for filler audio")
+        return v
+
+
+class AgentConfig(TypedModel, type=AgentType.BASE):
+    initial_message: Optional[BaseMessage] = None
+    generate_responses: bool = True
+    allowed_idle_time_seconds: Optional[float] = None
+    end_conversation_on_goodbye: bool = False
+    send_filler_audio: Union[bool, FillerAudioConfig] = False
+
+
+class CutOffResponse(BaseModel):
+    messages: list[BaseMessage] = [BaseMessage(text="Sorry?")]
+
+
+class LLMAgentConfig(AgentConfig, type=AgentType.LLM):
+    prompt_preamble: str
+    expected_first_prompt: Optional[str] = None
+    model_name: str = LLM_AGENT_DEFAULT_MODEL_NAME
+    temperature: float = LLM_AGENT_DEFAULT_TEMPERATURE
+    max_tokens: int = LLM_AGENT_DEFAULT_MAX_TOKENS
+    cut_off_response: Optional[CutOffResponse] = None
+
+
+class ChatGPTAgentConfig(AgentConfig, type=AgentType.CHAT_GPT):
+    prompt_preamble: str
+    expected_first_prompt: Optional[str] = None
+    generate_responses: bool = False
+    model_name: str = CHAT_GPT_AGENT_DEFAULT_MODEL_NAME
+    temperature: float = LLM_AGENT_DEFAULT_TEMPERATURE
+    max_tokens: int = LLM_AGENT_DEFAULT_MAX_TOKENS
+    cut_off_response: Optional[CutOffResponse] = None
+
+
+class InformationRetrievalAgentConfig(
+    AgentConfig, type=AgentType.INFORMATION_RETRIEVAL
+):
+    recipient_descriptor: str
+    caller_descriptor: str
+    goal_description: str
+    fields: list[str]
+    # TODO: add fields for IVR, voicemail
+
+
+class EchoAgentConfig(AgentConfig, type=AgentType.ECHO):
+    pass
+
+
+class RESTfulUserImplementedAgentConfig(
+    AgentConfig, type=AgentType.RESTFUL_USER_IMPLEMENTED
+):
+    class EndpointConfig(BaseModel):
+        url: str
+        method: str = "POST"
+
+    respond: EndpointConfig
+    generate_responses: bool = False
+    # generate_response: Optional[EndpointConfig]
+    # update_last_bot_message_on_cut_off: Optional[EndpointConfig]
+
+
+class RESTfulAgentInput(BaseModel):
+    conversation_id: str
+    human_input: str
+
+
+class RESTfulAgentOutputType(str, Enum):
+    BASE = "restful_agent_base"
+    TEXT = "restful_agent_text"
+    END = "restful_agent_end"
+
+
+class RESTfulAgentOutput(TypedModel, type=RESTfulAgentOutputType.BASE):
+    pass
+
+
+class RESTfulAgentText(RESTfulAgentOutput, type=RESTfulAgentOutputType.TEXT):
+    response: str
+
+
+class RESTfulAgentEnd(RESTfulAgentOutput, type=RESTfulAgentOutputType.END):
+    pass
+
+
+class WebSocketUserImplementedAgentConfig(
+    AgentConfig, type=AgentType.WEBSOCKET_USER_IMPLEMENTED
+):
+    class RouteConfig(BaseModel):
+        url: str
+
+    respond: RouteConfig
+    generate_responses: bool = False
+    # generate_response: Optional[RouteConfig]
+    # send_message_on_cut_off: bool = False
+
+
+class WebSocketAgentMessageType(str, Enum):
+    BASE = "websocket_agent_base"
+    START = "websocket_agent_start"
+    TEXT = "websocket_agent_text"
+    TEXT_END = "websocket_agent_text_end"
+    READY = "websocket_agent_ready"
+    STOP = "websocket_agent_stop"
+
+
+class WebSocketAgentMessage(TypedModel, type=WebSocketAgentMessageType.BASE):
+    conversation_id: Optional[str] = None
+
+
+class WebSocketAgentTextMessage(
+    WebSocketAgentMessage, type=WebSocketAgentMessageType.TEXT
+):
+    class Payload(BaseModel):
+        text: str
+
+    data: Payload
+
+    @classmethod
+    def from_text(cls, text: str, conversation_id: Optional[str] = None):
+        return cls(data=cls.Payload(text=text), conversation_id=conversation_id)
+
+
+class WebSocketAgentStartMessage(
+    WebSocketAgentMessage, type=WebSocketAgentMessageType.START
+):
+    pass
+
+
+class WebSocketAgentReadyMessage(
+    WebSocketAgentMessage, type=WebSocketAgentMessageType.READY
+):
+    pass
+
+
+class WebSocketAgentStopMessage(
+    WebSocketAgentMessage, type=WebSocketAgentMessageType.STOP
+):
+    pass
+
+
+class WebSocketAgentTextEndMessage(
+    WebSocketAgentMessage, type=WebSocketAgentMessageType.TEXT_END
+):
+    pass

vocode/streaming/models/audio_encoding.py

@@ -0,0 +1,5 @@
+from enum import Enum
+
+class AudioEncoding(str, Enum):
+    LINEAR16 = "linear16"
+    MULAW = "mulaw"

vocode/streaming/models/message.py

@@ -0,0 +1,16 @@
+from enum import Enum
+from .model import TypedModel
+from enum import Enum
+
+
+class MessageType(str, Enum):
+    BASE = "message_base"
+    SSML = "message_ssml"
+
+
+class BaseMessage(TypedModel, type=MessageType.BASE):
+    text: str
+
+
+class SSMLMessage(BaseMessage, type=MessageType.SSML):
+    ssml: str

vocode/streaming/models/model.py

@@ -0,0 +1,52 @@
+import pydantic
+
+class BaseModel(pydantic.BaseModel):
+
+    def __init__(self, **data):
+        for key, value in data.items():
+            if isinstance(value, dict):
+                if 'type' in value:
+                    data[key] = TypedModel.parse_obj(value)
+        super().__init__(**data)
+
+# Adapted from https://github.com/pydantic/pydantic/discussions/3091
+class TypedModel(BaseModel):
+
+    _subtypes_ = []
+
+    def __init_subclass__(cls, type=None):
+        cls._subtypes_.append([type, cls])
+
+    @classmethod
+    def get_cls(_cls, type):
+        for t, cls in _cls._subtypes_:
+            if t == type:
+                return cls
+        raise ValueError(f'Unknown type {type}')
+    
+    @classmethod
+    def get_type(_cls, cls_name):
+        for t, cls in _cls._subtypes_:
+            if cls.__name__ == cls_name:
+                return t
+        raise ValueError(f'Unknown class {cls_name}')
+    
+    @classmethod
+    def parse_obj(cls, obj):
+        data_type = obj.get('type')
+        if data_type is None:
+            raise ValueError(f'type is required for {cls.__name__}')
+    
+        sub = cls.get_cls(data_type)
+        if sub is None:
+            raise ValueError(f'Unknown type {data_type}')
+        return sub(**obj)
+
+    def _iter(self, **kwargs):
+        yield 'type', self.get_type(self.__class__.__name__)
+        yield from super()._iter(**kwargs)
+
+    @property
+    def type(self):
+        return self.get_type(self.__class__.__name__)
+

vocode/streaming/models/synthesizer.py

@@ -0,0 +1,73 @@
+from enum import Enum
+from typing import Optional, Union
+
+from pydantic import BaseModel, validator
+from .model import TypedModel
+from .audio_encoding import AudioEncoding
+from ..output_device.base_output_device import BaseOutputDevice
+
+
+class SynthesizerType(str, Enum):
+    BASE = "synthesizer_base"
+    AZURE = "synthesizer_azure"
+    GOOGLE = "synthesizer_google"
+    ELEVEN_LABS = "synthesizer_eleven_labs"
+
+
+class TrackBotSentimentConfig(BaseModel):
+    emotions: list[str] = ["angry", "friendly", "sad", "whispering"]
+
+    @validator("emotions")
+    def emotions_must_not_be_empty(cls, v):
+        if len(v) == 0:
+            raise ValueError("must have at least one emotion")
+        return v
+
+
+class SynthesizerConfig(TypedModel, type=SynthesizerType.BASE):
+    sampling_rate: int
+    audio_encoding: AudioEncoding
+    should_encode_as_wav: bool = False
+    track_bot_sentiment_in_voice: Union[bool, TrackBotSentimentConfig] = False
+
+    @classmethod
+    def from_output_device(cls, output_device: BaseOutputDevice):
+        return cls(
+            sampling_rate=output_device.sampling_rate,
+            audio_encoding=output_device.audio_encoding,
+        )
+
+
+AZURE_SYNTHESIZER_DEFAULT_VOICE_NAME = "en-US-AriaNeural"
+AZURE_SYNTHESIZER_DEFAULT_PITCH = 0
+AZURE_SYNTHESIZER_DEFAULT_RATE = 15
+
+
+class AzureSynthesizerConfig(SynthesizerConfig, type=SynthesizerType.AZURE):
+    voice_name: str = AZURE_SYNTHESIZER_DEFAULT_VOICE_NAME
+    pitch: int = AZURE_SYNTHESIZER_DEFAULT_PITCH
+    rate: int = AZURE_SYNTHESIZER_DEFAULT_RATE
+
+    @classmethod
+    def from_output_device(
+        cls,
+        output_device: BaseOutputDevice,
+        voice_name: str = AZURE_SYNTHESIZER_DEFAULT_VOICE_NAME,
+        pitch: int = AZURE_SYNTHESIZER_DEFAULT_PITCH,
+        rate: int = AZURE_SYNTHESIZER_DEFAULT_RATE,
+        track_bot_sentiment_in_voice: Union[bool, TrackBotSentimentConfig] = False,
+    ):
+        return cls(
+            sampling_rate=output_device.sampling_rate,
+            audio_encoding=output_device.audio_encoding,
+            voice_name=voice_name,
+            pitch=pitch,
+            rate=rate,
+            track_bot_sentiment_in_voice=track_bot_sentiment_in_voice,
+        )
+
+    pass
+
+
+class GoogleSynthesizerConfig(SynthesizerConfig, type=SynthesizerType.GOOGLE):
+    pass

vocode/streaming/models/telephony.py

@@ -0,0 +1,50 @@
+from typing import Optional
+from vocode.streaming.models.model import BaseModel
+from vocode.streaming.models.agent import AgentConfig
+from vocode.streaming.models.synthesizer import SynthesizerConfig
+from vocode.streaming.models.transcriber import TranscriberConfig
+
+
+class TwilioConfig(BaseModel):
+    account_sid: str
+    auth_token: str
+
+
+class CallEntity(BaseModel):
+    phone_number: str
+
+
+class CreateInboundCall(BaseModel):
+    transcriber_config: Optional[TranscriberConfig] = None
+    agent_config: AgentConfig
+    synthesizer_config: Optional[SynthesizerConfig] = None
+    twilio_sid: str
+    twilio_config: Optional[TwilioConfig] = None
+
+
+class EndOutboundCall(BaseModel):
+    call_id: str
+    twilio_config: Optional[TwilioConfig] = None
+
+
+class CreateOutboundCall(BaseModel):
+    recipient: CallEntity
+    caller: CallEntity
+    transcriber_config: Optional[TranscriberConfig] = None
+    agent_config: AgentConfig
+    synthesizer_config: Optional[SynthesizerConfig] = None
+    conversation_id: Optional[str] = None
+    twilio_config: Optional[TwilioConfig] = None
+    # TODO add IVR/etc.
+
+
+class DialIntoZoomCall(BaseModel):
+    recipient: CallEntity
+    caller: CallEntity
+    zoom_meeting_id: str
+    zoom_meeting_password: Optional[str]
+    transcriber_config: Optional[TranscriberConfig] = None
+    agent_config: AgentConfig
+    synthesizer_config: Optional[SynthesizerConfig] = None
+    conversation_id: Optional[str] = None
+    twilio_config: Optional[TwilioConfig] = None

vocode/streaming/models/transcriber.py

@@ -0,0 +1,70 @@
+from enum import Enum
+from typing import Optional
+
+from vocode.streaming.input_device.base_input_device import (
+    BaseInputDevice,
+)
+from .audio_encoding import AudioEncoding
+from .model import BaseModel, TypedModel
+
+
+class TranscriberType(str, Enum):
+    BASE = "transcriber_base"
+    DEEPGRAM = "transcriber_deepgram"
+    GOOGLE = "transcriber_google"
+    ASSEMBLY_AI = "transcriber_assembly_ai"
+
+
+class EndpointingType(str, Enum):
+    BASE = "endpointing_base"
+    TIME_BASED = "endpointing_time_based"
+    PUNCTUATION_BASED = "endpointing_punctuation_based"
+
+
+class EndpointingConfig(TypedModel, type=EndpointingType.BASE):
+    pass
+
+
+class TimeEndpointingConfig(EndpointingConfig, type=EndpointingType.TIME_BASED):
+    time_cutoff_seconds: float = 0.4
+
+
+class PunctuationEndpointingConfig(
+    EndpointingConfig, type=EndpointingType.PUNCTUATION_BASED
+):
+    time_cutoff_seconds: float = 0.4
+
+
+class TranscriberConfig(TypedModel, type=TranscriberType.BASE):
+    sampling_rate: int
+    audio_encoding: AudioEncoding
+    chunk_size: int
+    endpointing_config: Optional[EndpointingConfig] = None
+
+    @classmethod
+    def from_input_device(
+        cls,
+        input_device: BaseInputDevice,
+        endpointing_config: Optional[EndpointingConfig] = None,
+    ):
+        return cls(
+            sampling_rate=input_device.sampling_rate,
+            audio_encoding=input_device.audio_encoding,
+            chunk_size=input_device.chunk_size,
+            endpointing_config=endpointing_config,
+        )
+
+
+class DeepgramTranscriberConfig(TranscriberConfig, type=TranscriberType.DEEPGRAM):
+    model: Optional[str] = None
+    should_warmup_model: bool = False
+    version: Optional[str] = None
+
+
+class GoogleTranscriberConfig(TranscriberConfig, type=TranscriberType.GOOGLE):
+    model: Optional[str] = None
+    should_warmup_model: bool = False
+
+
+class AssemblyAITranscriberConfig(TranscriberConfig, type=TranscriberType.ASSEMBLY_AI):
+    should_warmup_model: bool = False

vocode/streaming/models/websocket.py

@@ -0,0 +1,38 @@
+import base64
+from enum import Enum
+from typing import Optional
+from .model import TypedModel
+from .transcriber import TranscriberConfig
+from .agent import AgentConfig
+from .synthesizer import SynthesizerConfig
+
+class WebSocketMessageType(str, Enum):
+    BASE = 'websocket_base'
+    START = 'websocket_start'
+    AUDIO = 'websocket_audio'
+    READY = 'websocket_ready'
+    STOP = 'websocket_stop'
+
+class WebSocketMessage(TypedModel, type=WebSocketMessageType.BASE): pass
+
+class AudioMessage(WebSocketMessage, type=WebSocketMessageType.AUDIO):
+    data: str
+
+    @classmethod
+    def from_bytes(cls, chunk: bytes):
+        return cls(data=base64.b64encode(chunk).decode('utf-8'))
+
+    def get_bytes(self) -> bytes:
+        return base64.b64decode(self.data)
+
+class StartMessage(WebSocketMessage, type=WebSocketMessageType.START):
+    transcriber_config: TranscriberConfig
+    agent_config: AgentConfig
+    synthesizer_config: SynthesizerConfig
+    conversation_id: Optional[str] = None
+
+class ReadyMessage(WebSocketMessage, type=WebSocketMessageType.READY):
+    pass
+
+class StopMessage(WebSocketMessage, type=WebSocketMessageType.STOP):
+    pass