From 50bc04b9877e3a4db32a85e86278c93a1d002e17 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Tue, 22 Mar 2022 13:30:30 -0400 Subject: [PATCH] Fixing marks in CLI --- mimic3-cli/mimic3_cli/__main__.py | 432 +++++++++------------------- mimic3-http/mimic3_http/__main__.py | 18 +- mimic3-http/run.sh | 20 ++ mimic3-tts/mimic3_tts/__init__.py | 1 + mimic3-tts/mimic3_tts/tts.py | 391 ++----------------------- mimic3-tts/mimic3_tts/voice.py | 32 ++- opentts-abc/opentts_abc/ssml.py | 8 + 7 files changed, 237 insertions(+), 665 deletions(-) create mode 100755 mimic3-http/run.sh diff --git a/mimic3-cli/mimic3_cli/__main__.py b/mimic3-cli/mimic3_cli/__main__.py index fa6d44b..7614851 100644 --- a/mimic3-cli/mimic3_cli/__main__.py +++ b/mimic3-cli/mimic3_cli/__main__.py @@ -22,7 +22,7 @@ from pathlib import Path from queue import Queue if typing.TYPE_CHECKING: - from mimic3_tts import Mimic3TextToSpeechSystem + from mimic3_tts import Mimic3TextToSpeechSystem, BaseResult _DIR = Path(__file__).parent @@ -46,8 +46,15 @@ class CommandLineInterfaceState: sample_width_bytes: int = 2 num_channels: int = 1 - raw_queue: typing.Optional["Queue[typing.Optional[bytes]]"] = None - raw_stream_thread: typing.Optional[threading.Thread] = None + result_queue: typing.Optional["Queue[BaseResult]"] = None + result_thread: typing.Optional[threading.Thread] = None + + +@dataclass +class ResultToProcess: + result: "BaseResult" + line: str + line_id: str = "" class OutputNaming(str, Enum): @@ -133,8 +140,10 @@ def initialize_args(state: CommandLineInterfaceState): state.mark_writer = open( # pylint: disable=consider-using-with args.mark_file, "w", encoding="utf-8" ) - else: + elif args.stdout: state.mark_writer = sys.stderr + else: + state.mark_writer = sys.stdout if args.seed is not None: _LOGGER.debug("Setting random seed to %s", args.seed) @@ -201,181 +210,138 @@ def initialize_tts(state: CommandLineInterfaceState): args = state.args - # TODO: voice/speaker state.tts = Mimic3TextToSpeechSystem(Mimic3Settings()) if state.args.voice: + # Set default voice state.tts.voice = state.args.voice - # max_thread_workers: typing.Optional[int] = None + if state.args.preload_voice: + for voice_key in state.args.preload_voice: + _LOGGER.debug("Preloading voice: %s", voice_key) + state.tts.preload_voice(voice_key) - # if args.max_thread_workers is not None: - # max_thread_workers = ( - # None if args.max_thread_workers < 1 else args.max_thread_workers - # ) - # elif args.raw_stream: - # # Faster time to first audio - # max_thread_workers = 2 + state.result_queue = Queue(maxsize=args.result_queue_size) - # executor = ThreadPoolExecutor(max_workers=max_thread_workers) - - # if os.isatty(sys.stdout.fileno()): - # if (not args.output_dir) and (not args.raw_stream): - # # No where else for the audio to go - # args.interactive = True - - if args.raw_stream: - # Output in a separate thread to avoid blocking audio processing - state.raw_queue = Queue(maxsize=args.raw_stream_queue_size) - - def output_raw_stream(): - while True: - audio = state.raw_queue.get() - if audio is None: - break - - _LOGGER.debug( - "Writing %s byte(s) of 16-bit 22050Hz mono PCM to stdout", - len(audio), - ) - sys.stdout.buffer.write(audio) - sys.stdout.buffer.flush() - - state.raw_stream_thread = threading.Thread( - target=output_raw_stream, daemon=True - ) - state.raw_stream_thread.start() + state.result_thread = threading.Thread( + target=process_result, daemon=True, args=(state,) + ) + state.result_thread.start() -def process_line(line_id: str, line: str, state: CommandLineInterfaceState): - from mimic3_tts import AudioResult, MarkResult +def process_result(state: CommandLineInterfaceState): + try: + from mimic3_tts import AudioResult, MarkResult + + assert state.result_queue is not None + args = state.args + + while True: + result_todo = state.result_queue.get() + if result_todo is None: + break + + try: + result = result_todo.result + line = result_todo.line + line_id = result_todo.line_id + + if isinstance(result, AudioResult): + if args.interactive or args.output_dir: + # Convert to WAV audio + wav_bytes: typing.Optional[bytes] = None + if args.interactive: + if args.stdout: + # Write audio to stdout + sys.stdout.buffer.write(result.audio_bytes) + sys.stdout.buffer.flush() + else: + # Play sound + if not wav_bytes: + wav_bytes = result.to_wav_bytes() + + if wav_bytes: + play_wav_bytes(wav_bytes) + + if args.output_dir: + if not wav_bytes: + wav_bytes = result.to_wav_bytes() + + # Determine file name + if args.output_naming == OutputNaming.TEXT: + # Use text itself + file_name = line.strip().replace(" ", "_") + file_name = file_name.translate( + str.maketrans( + "", "", string.punctuation.replace("_", "") + ) + ) + elif args.output_naming == OutputNaming.TIME: + # Use timestamp + file_name = str(time.time()) + elif args.output_naming == OutputNaming.ID: + file_name = line_id + + assert file_name, f"No file name for text: {line}" + wav_path = args.output_dir / (file_name + ".wav") + wav_path.write_bytes(wav_bytes) + + _LOGGER.debug("Wrote %s", wav_path) + else: + # Combine all audio and output to stdout at the end + state.all_audio += result.audio_bytes + state.sample_rate_hz = result.sample_rate_hz + state.sample_width_bytes = result.sample_width_bytes + state.num_channels = result.num_channels + elif isinstance(result, MarkResult): + if state.mark_writer: + print(result.name, file=state.mark_writer) + except Exception: + _LOGGER.exception("Error processing result") + except Exception: + _LOGGER.exception("process_result") + + +def process_line( + line: str, + state: CommandLineInterfaceState, + line_id: str = "", +): + from mimic3_tts import SSMLSpeaker + + assert state.tts is not None + assert state.result_queue is not None args = state.args - assert state.tts is not None - # TODO: SSML - state.tts.begin_utterance() + if args.ssml: + results = SSMLSpeaker(state.tts).speak(line) + else: + state.tts.begin_utterance() - # TODO: text language - state.tts.speak_text(line) + # TODO: text language + state.tts.speak_text(line) - # TODO: CSV - text_id = "" - result_idx = 0 + results = state.tts.end_utterance() - for result in state.tts.end_utterance(): - if isinstance(result, AudioResult): - if args.raw_stream: - assert state.raw_queue is not None - state.raw_queue.put(result.audio_bytes) - elif args.interactive or args.output_dir: - # Convert to WAV audio - wav_bytes: typing.Optional[bytes] = None - if args.interactive: - if not wav_bytes: - wav_bytes = result.to_wav_bytes() - - play_wav_bytes(wav_bytes) - - if args.output_dir: - if not wav_bytes: - wav_bytes = result.to_wav_bytes() - - # Determine file name - if args.output_naming == OutputNaming.TEXT: - # Use text itself - file_name = line.strip().replace(" ", "_") - file_name = file_name.translate( - str.maketrans("", "", string.punctuation.replace("_", "")) - ) - elif args.output_naming == OutputNaming.TIME: - # Use timestamp - file_name = str(time.time()) - elif args.output_naming == OutputNaming.ID: - if not text_id: - text_id = line_id - else: - text_id = f"{line_id}_{result_idx + 1}" - - file_name = text_id - - assert file_name, f"No file name for text: {line}" - wav_path = args.output_dir / (file_name + ".wav") - wav_path.write_bytes(wav_bytes) - - _LOGGER.debug("Wrote %s", wav_path) - else: - # Combine all audio and output to stdout at the end - state.all_audio += result.audio_bytes - state.sample_rate_hz = result.sample_rate_hz - state.sample_width_bytes = result.sample_width_bytes - state.num_channels = result.num_channels - - result_idx += 1 - elif isinstance(result, MarkResult): - if state.mark_writer: - print(result.name, file=state.mark_writer) - - # text_id = "" - - # for result_idx, result in enumerate(tts_results): - # text = result.text - - # # Write before marks - # if result.marks_before and state.mark_writer: - # for mark_name in result.marks_before: - # print(mark_name, file=state.mark_writer) - - # if args.raw_stream: - # assert raw_queue is not None - # raw_queue.put(result.audio.tobytes()) - # elif args.interactive or args.output_dir: - # # Convert to WAV audio - # with io.BytesIO() as wav_io: - # wav_write(wav_io, result.sample_rate, result.audio) - # wav_data = wav_io.getvalue() - - # assert wav_data is not None - - # if args.interactive: - - # # Play audio - # _LOGGER.debug("Playing audio with play command") - # try: - # subprocess.run( - # play_command, - # input=wav_data, - # stdout=subprocess.DEVNULL, - # stderr=subprocess.DEVNULL, - # check=True, - # ) - # except FileNotFoundError: - # _LOGGER.error( - # "Unable to play audio with command '%s'. set with --play-command or redirect stdout", - # args.play_command, - # ) - # with open("output.wav", "wb") as output_file: - # output_file.write(wav_data) - - # _LOGGER.warning("stdout not redirected. Wrote audio to output.wav.") - - # else: - # # Combine all audio and output to stdout at the end - # all_audios.append(result.audio) - - # # Write after marks - # if result.marks_after and state.mark_writer: - # for mark_name in result.marks_after: - # print(mark_name, file=state.mark_writer) + for result in results: + state.result_queue.put( + ResultToProcess( + result=result, + line=line, + line_id=line_id, + ) + ) def process_lines(state: CommandLineInterfaceState): assert state.texts is not None args = state.args - start_time_to_first_audio = time.perf_counter() try: + result_idx = 0 + for line in state.texts: line_id = "" line = line.strip() @@ -386,20 +352,22 @@ def process_lines(state: CommandLineInterfaceState): # Line has the format id|text instead of just text line_id, line = line.split(args.id_delimiter, maxsplit=1) - process_line(line_id, line, state) + process_line(line, state, line_id=line_id) + result_idx += 1 except KeyboardInterrupt: - if state.raw_queue is not None: + if state.result_queue is not None: # Draw audio playback queue - while not state.raw_queue.empty(): - state.raw_queue.get() + while not state.result_queue.empty(): + state.result_queue.get() finally: # Wait for raw stream to finish - if state.raw_queue is not None: - state.raw_queue.put(None) + if state.result_queue is not None: + state.result_queue.put(None) - if state.raw_stream_thread is not None: - state.raw_stream_thread.join() + if state.result_thread is not None: + print("Waiting for audio to finish...", file=sys.stderr) + state.result_thread.join() # ------------------------------------------------------------------------- @@ -502,37 +470,24 @@ def get_args(): parser.add_argument( "--noise-scale", type=float, - default=0.333, - help="Noise scale (default: 0.333)", + help="Noise scale [0-1], default is 0.667", ) parser.add_argument( "--length-scale", type=float, - default=1.0, - help="Length scale (default: 1.0)", + help="Length scale (1.0 is default speed, 0.5 is 2x faster)", ) parser.add_argument( "--noise-w", type=float, - default=1.0, - help="Variation in cadence (default: 1.0)", + help="Variation in cadence [0-1], default is 0.8", ) # Miscellaneous parser.add_argument( - "--max-thread-workers", - type=int, - help="Maximum number of threads to concurrently load models and run sentences through TTS/Vocoder", - ) - parser.add_argument( - "--raw-stream", - action="store_true", - help="Stream raw 16-bit 22050Hz mono PCM audio to stdout", - ) - parser.add_argument( - "--raw-stream-queue-size", + "--result-queue-size", default=5, - help="Maximum number of sentences to maintain in output queue with --raw-stream (default: 5)", + help="Maximum number of sentences to maintain in output queue (default: 5)", ) parser.add_argument( "--process-on-blank-line", @@ -552,6 +507,9 @@ def get_args(): action="store_true", help="Force audio output to stdout even if a tty is detected", ) + parser.add_argument( + "--preload-voice", action="append", help="Preload voice when starting up" + ) parser.add_argument("--seed", type=int, help="Set random seed (default: not set)") # parser.add_argument("--version", action="store_true", help="Print version and exit") parser.add_argument( @@ -575,116 +533,6 @@ def get_args(): # ------------------------------------------------------------------------- - # # Directories to search for voices - # voices_dirs = get_voices_dirs(args.voices_dir) - - # def list_voices_vocoders(): - # """Print all vocoders and voices""" - # # (type, name) -> location - # local_info = {} - - # # Search for downloaded voices/vocoders - # for voices_dir in voices_dirs: - # if not voices_dir.is_dir(): - # continue - - # for voice_dir in voices_dir.iterdir(): - # if not voice_dir.is_dir(): - # continue - - # if voice_dir.name in VOCODER_DIR_NAMES: - # # Vocoder - # for vocoder_model_dir in voice_dir.iterdir(): - # if not valid_voice_dir(vocoder_model_dir): - # continue - - # full_vocoder_name = f"{voice_dir.name}-{vocoder_model_dir.name}" - # local_info[("vocoder", full_vocoder_name)] = str( - # vocoder_model_dir - # ) - # else: - # # Voice - # voice_lang = voice_dir.name - # for voice_model_dir in voice_dir.iterdir(): - # if not valid_voice_dir(voice_model_dir): - # continue - - # local_info[("voice", voice_model_dir.name)] = str( - # voice_model_dir - # ) - - # # (type, lang, name, downloaded, aliases, location) - # voices_and_vocoders = [] - # with open(_DIR / "VOCODERS", "r", encoding="utf-8") as vocoders_file: - # for line in vocoders_file: - # line = line.strip() - # if not line: - # continue - - # *vocoder_aliases, full_vocoder_name = line.split() - # downloaded = False - - # location = local_info.get(("vocoder", full_vocoder_name), "") - # if location: - # downloaded = True - - # voices_and_vocoders.append( - # ( - # "vocoder", - # " ", - # "*" if downloaded else " ", - # full_vocoder_name, - # ",".join(vocoder_aliases), - # location, - # ) - # ) - - # with open(_DIR / "VOICES", "r", encoding="utf-8") as voices_file: - # for line in voices_file: - # line = line.strip() - # if not line: - # continue - - # *voice_aliases, full_voice_name, download_name = line.split() - # voice_lang = download_name.split("_", maxsplit=1)[0] - - # downloaded = False - - # location = local_info.get(("voice", full_voice_name), "") - # if location: - # downloaded = True - - # voices_and_vocoders.append( - # ( - # "voice", - # voice_lang, - # "*" if downloaded else " ", - # full_voice_name, - # ",".join(voice_aliases), - # location, - # ) - # ) - - # headers = ("TYPE", "LANG", "LOCAL", "NAME", "ALIASES", "LOCATION") - - # # Get widths of columns - # col_widths = [0] * len(voices_and_vocoders[0]) - # for item in voices_and_vocoders: - # for col in range(len(col_widths)): - # col_widths[col] = max( - # col_widths[col], len(item[col]) + 1, len(headers[col]) + 1 - # ) - - # # Print results - # print(*(h.ljust(col_widths[col]) for col, h in enumerate(headers))) - - # for item in sorted(voices_and_vocoders): - # print(*(v.ljust(col_widths[col]) for col, v in enumerate(item))) - - # if args.list: - # list_voices_vocoders() - # sys.exit(0) - return args diff --git a/mimic3-http/mimic3_http/__main__.py b/mimic3-http/mimic3_http/__main__.py index 58e3074..3feac96 100644 --- a/mimic3-http/mimic3_http/__main__.py +++ b/mimic3-http/mimic3_http/__main__.py @@ -15,13 +15,13 @@ # import argparse import asyncio +import dataclasses +import io import logging import sys -import io -import wave import tempfile import typing -import dataclasses +import wave from dataclasses import dataclass from pathlib import Path from urllib.parse import parse_qs @@ -29,6 +29,7 @@ from uuid import uuid4 import hypercorn import quart_cors +from mimic3_tts import AudioResult, Mimic3Settings, Mimic3TextToSpeechSystem from quart import ( Quart, Response, @@ -38,8 +39,6 @@ from quart import ( send_from_directory, ) -from mimic3_tts import Mimic3TextToSpeechSystem, Mimic3Settings, AudioResult - _LOGGER = logging.getLogger(__name__) _MISSING = object() @@ -79,6 +78,9 @@ parser.add_argument( default=_MISSING, help="Enable WAV cache with optional directory (default: no cache)", ) +parser.add_argument( + "--preload-voice", action="append", help="Preload voice when starting up" +) # parser.add_argument( # "--max-loaded-models", # type=int, @@ -132,7 +134,6 @@ _WAV_CACHE: typing.Dict[TextToWavParams, Path] = {} # TODO: XDG voice directories # TODO: args.voices_dir -# TODO: Preload voice mimic3 = Mimic3TextToSpeechSystem( Mimic3Settings( @@ -144,6 +145,11 @@ mimic3 = Mimic3TextToSpeechSystem( ) ) +if args.preload_voice: + for voice_key in args.preload_voice: + _LOGGER.debug("Preloading voice: %s", voice_key) + mimic3.preload_voice(voice_key) + def text_to_wav(params: TextToWavParams, no_cache: bool = False) -> bytes: diff --git a/mimic3-http/run.sh b/mimic3-http/run.sh new file mode 100755 index 0000000..9f59531 --- /dev/null +++ b/mimic3-http/run.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Directory of *this* script +this_dir="$( cd "$( dirname "$0" )" && pwd )" + +# Kebab to snake case +module_name="$(basename "${this_dir}" | sed -e 's/-/_/g')" +src_dir="${this_dir}/${module_name}" + +# Path to virtual environment +: "${venv:=${this_dir}/.venv}" + +if [ -d "${venv}" ]; then + # Activate virtual environment if available + source "${venv}/bin/activate" +fi + +export PYTHONPATH="${this_dir}" +python3 -m "${module_name}" "$@" diff --git a/mimic3-tts/mimic3_tts/__init__.py b/mimic3-tts/mimic3_tts/__init__.py index 087a94a..a0b8eac 100644 --- a/mimic3-tts/mimic3_tts/__init__.py +++ b/mimic3-tts/mimic3_tts/__init__.py @@ -1,3 +1,4 @@ from opentts_abc import AudioResult, MarkResult +from opentts_abc.ssml import SSMLSpeaker from .tts import Mimic3Settings, Mimic3TextToSpeechSystem diff --git a/mimic3-tts/mimic3_tts/tts.py b/mimic3-tts/mimic3_tts/tts.py index 2fbf85f..585af9b 100644 --- a/mimic3-tts/mimic3_tts/tts.py +++ b/mimic3-tts/mimic3_tts/tts.py @@ -26,7 +26,6 @@ from opentts_abc import ( ) from mimic3_tts.config import TrainingConfig -from mimic3_tts.utils import audio_float_to_int16 from mimic3_tts.voice import Mimic3Voice, SPEAKER_TYPE _DIR = Path(__file__).parent @@ -48,9 +47,9 @@ class Mimic3Settings: language: typing.Optional[str] = None voices_directories: typing.Optional[typing.Iterable[typing.Union[str, Path]]] = None speaker: typing.Optional[SPEAKER_TYPE] = None - length_scale: float = 1.0 - noise_scale: float = 0.667 - noise_w: float = 0.8 + length_scale: typing.Optional[float] = None + noise_scale: typing.Optional[float] = None + noise_w: typing.Optional[float] = None text_language: typing.Optional[str] = None sample_rate: int = 22050 @@ -59,6 +58,12 @@ class Mimic3Settings: class Mimic3Phonemes: current_settings: Mimic3Settings phonemes: typing.List[typing.List[str]] = field(default_factory=list) + is_utterance: bool = True + + +class VoiceNotFoundError(Exception): + def __init__(self, voice: str): + super().__init__(f"Voice not found: {voice}") # ----------------------------------------------------------------------------- @@ -71,8 +76,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): self.settings = settings self._results: typing.List[typing.Union[BaseResult, Mimic3Phonemes]] = [] - - self.loaded_voices: typing.Dict[str, Mimic3Voice] = {} + self._loaded_voices: typing.Dict[str, Mimic3Voice] = {} @property def voice(self) -> str: @@ -173,7 +177,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): ) def begin_utterance(self): - self._results.clear() + pass def speak_text(self, text: str, text_language: typing.Optional[str] = None): voice = self._get_or_load_voice(self.voice) @@ -275,7 +279,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): for result in self._results: if isinstance(result, Mimic3Phonemes): - if result.current_settings != last_settings: + if result.is_utterance or (result.current_settings != last_settings): if sent_phonemes: yield self._speak_sentence_phonemes( sent_phonemes, settings=last_settings @@ -295,9 +299,15 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): if sent_phonemes: yield self._speak_sentence_phonemes(sent_phonemes) + sent_phonemes.clear() + + self._results.clear() + + def preload_voice(self, voice_key: str): + self._get_or_load_voice(voice_key) def _get_or_load_voice(self, voice_key: str) -> Mimic3Voice: - existing_voice = self.loaded_voices.get(voice_key) + existing_voice = self._loaded_voices.get(voice_key) if existing_voice is not None: return existing_voice @@ -308,11 +318,16 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): model_dir = Path(maybe_voice.key) break - assert model_dir is not None - existing_voice = self.loaded_voices.get(str(model_dir.absolute())) + if model_dir is None: + raise VoiceNotFoundError(voice_key) + + # Full path to voice model directory + canonical_key = str(model_dir.absolute()) + + existing_voice = self._loaded_voices.get(canonical_key) if existing_voice is not None: # Alias - self.loaded_voices[voice_key] = existing_voice + self._loaded_voices[voice_key] = existing_voice return existing_voice @@ -321,355 +336,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): _LOGGER.info("Loaded voice from %s", model_dir) # Add to cache - self.loaded_voices[voice_key] = voice + self._loaded_voices[voice_key] = voice + self._loaded_voices[canonical_key] = voice return voice - - # def start(self): - # self.stop() - - # self._thread = threading.Thread(target=self._thread_proc, daemon=True) - # self._thread.start() - - # def stop(self): - # if self._thread is not None: - # self._request_queue.put(None) - # self._thread.join() - # self._thread = None - - # # Drain queues - # while not self._request_queue.empty(): - # self._request_queue.get() - - # while not self._result_queue.empty(): - # self._result_queue.get() - - # def _thread_proc(self): - # try: - # self._load_model() - # self._load_text_processor() - - # while True: - # message = self._request_queue.get() - # if message is None: - # break - - # if isinstance(message, AddLexiconMessage): - # self._add_lexicon(message.lexicon_file) - # elif isinstance(message, TextToSpeechMessage): - # result = self._text_to_speech(**dataclasses.asdict(message)) - # self._result_queue.put(result) - - # except Exception: - # _LOGGER.exception("_thread_proc") - - # def _load_model(self): - # """Load model configuration and generator""" - - # if self._config is None: - # config_path = self.model_dir / "config.json" - # _LOGGER.debug("Loading model config from %s", config_path) - - # with open(config_path, "r", encoding="utf-8") as config_file: - # self._config = TrainingConfig.load(config_file) - - # self.lang = self.lang or self._config.text_language or "en_US" - - # if self._phoneme_to_id is None: - # # phoneme -> id - # phoneme_ids_path = self.model_dir / "phonemes.txt" - # _LOGGER.debug("Loading model phonemes from %s", phoneme_ids_path) - # with open(phoneme_ids_path, "r", encoding="utf-8") as ids_file: - # self._phoneme_to_id = phonemes2ids.load_phoneme_ids(ids_file) - - # valid_phonemes = [] - # for phoneme_str in self._phoneme_to_id: - # maybe_phoneme = Phoneme(phoneme_str) - # if any( - # [ - # maybe_phoneme.vowel, - # maybe_phoneme.consonant, - # maybe_phoneme.dipthong, - # maybe_phoneme.schwa, - # ] - # ): - # valid_phonemes.append(maybe_phoneme) - - # self._voice_phonemes = Phonemes(phonemes=valid_phonemes) - - # if self._phoneme_map is None: - # # phoneme -> phoneme, phoneme, ... - # phoneme_map_path = self.model_dir / "phoneme_map.txt" - # if phoneme_map_path.is_file(): - # _LOGGER.debug("Loading phoneme map from %s", phoneme_map_path) - # with open(phoneme_map_path, "r", encoding="utf-8") as map_file: - # self._phoneme_map = phonemes2ids.utils.load_phoneme_map(map_file) - - # if self._onnx_model is None: - # generator_path = self.model_dir / "generator.onnx" - # _LOGGER.debug("Loading model from %s", generator_path) - - # sess_options = onnxruntime.SessionOptions() - # sess_options.enable_cpu_mem_arena = False - # sess_options.enable_mem_pattern = False - # sess_options.enable_mem_reuse = False - - # self._onnx_model = onnxruntime.InferenceSession( - # str(generator_path), sess_options=sess_options - # ) - - # def _load_text_processor(self): - # if self._text_processor is None: - # self._text_processor = gruut.TextProcessor(default_lang=self.lang) - - # def add_lexicon(self, lexicon_file: typing.Iterable[str]): - # """Load a custom pronunciation lexicon from a file. - - # Format is: - # ... - - # Role can be things like "gruut:VB" or "gruut:NN". - # Use "_" for the default role (any part of speech). - # """ - # self._request_queue.put(AddLexiconMessage(lexicon_file=list(lexicon_file))) - - # def _add_lexicon(self, lexicon_file: typing.Iterable[str]): - # self._load_text_processor() - # assert self._text_processor is not None - - # # word -> role -> [phoneme, phoneme, ...] - # lexicon: typing.Dict[str, typing.Dict[str, typing.List[str]]] = {} - - # for line in lexicon_file: - # line = line.strip() - # if not line: - # continue - - # word, role, *phonemes = line.split() - # if (not word) or (not phonemes): - # _LOGGER.warning("Empty word or pronunciation in lexicon: %s", line) - # continue - - # if role == "_": - # role = WordRole.DEFAULT - - # word_roles = lexicon.get(word) - # if word_roles is None: - # word_roles = {} - # lexicon[word] = word_roles - - # word_roles[role] = phonemes - - # if lexicon: - - # # Wrap the "lookup_phonemes" method in the gruut text processor. - # # Our lexicon will be consulted first. - # settings = self._text_processor.get_settings() - # base_lookup = settings.lookup_phonemes - - # def lookup_phonemes(word: str, role: typing.Optional[str] = None, **kwargs): - # word_roles = lexicon.get(word) - - # if not word_roles: - # # Try lower case - # word_roles = lexicon.get(word.lower()) - - # if word_roles: - # if role is None: - # role = WordRole.DEFAULT - - # phonemes = word_roles.get(role) - - # if (phonemes is None) and (role != WordRole.DEFAULT): - # phonemes = word_roles.get(WordRole.DEFAULT) - - # if phonemes: - # return phonemes - - # if base_lookup is not None: - # return base_lookup(word, role, **kwargs) - - # return None - - # settings.lookup_phonemes = typing.cast(LookupPhonemes, lookup_phonemes) - # _LOGGER.debug("Added custom pronunciations for %s word(s)", len(lexicon)) - - # def text_to_speech( - # self, - # text: str, - # speaker_id: typing.Optional[int] = None, - # length_scale: typing.Optional[float] = None, - # noise_scale: typing.Optional[float] = None, - # noise_w: typing.Optional[float] = None, - # ssml: bool = False, - # text_language: typing.Optional[str] = None, - # ) -> Result: - # self._request_queue.put( - # TextToSpeechMessage( - # text=text, - # speaker_id=speaker_id, - # length_scale=length_scale, - # noise_scale=noise_scale, - # noise_w=noise_w, - # ssml=ssml, - # text_language=text_language, - # ) - # ) - - # result = typing.cast(Result, self._result_queue.get()) - - # return result - - # def _text_to_speech( - # self, - # text: str, - # speaker_id: typing.Optional[int] = None, - # length_scale: typing.Optional[float] = None, - # noise_scale: typing.Optional[float] = None, - # noise_w: typing.Optional[float] = None, - # ssml: bool = False, - # text_language: typing.Optional[str] = None, - # ) -> Result: - # """Speak text and return WAV audio as bytes""" - # text_language = text_language or self.lang - # assert self._text_processor is not None - - # # Ensure model is loaded - # assert self.lang is not None - # assert self._config is not None - # assert self._phoneme_to_id is not None - # assert self._onnx_model is not None - - # # Resolve settings - # if speaker_id is None: - # speaker_id = self.speaker_id or 0 - - # if length_scale is None: - # length_scale = self.length_scale - - # if noise_scale is None: - # noise_scale = self.noise_scale - - # if noise_w is None: - # noise_w = self.noise_w - - # # Process text into sentences - # result = Result(text=text) - # audio_arrays: typing.List[np.ndarray] = [] - - # graph, root = self._text_processor.process(text, lang=text_language, ssml=ssml) - # sentences = list(self._text_processor.sentences(graph, root)) - - # for sentence in sentences: - # result.sentence_words.append([w.text for w in sentence]) - - # if text_language == self.lang: - # sent_phonemes = [w.phonemes for w in sentence if w.phonemes] - # else: - # # Convert phonemes to ids to target language - # other_sent_phonemes = [w.phonemes for w in sentence if w.phonemes] - # _LOGGER.debug(other_sent_phonemes) - - # sent_phonemes = [] - # for other_word_p in other_sent_phonemes: - # word_p = [] - # for other_p in other_word_p: - # if IPA.is_break(other_p): - # # Keep breaks - # word_p.append(other_p) - # continue - - # original_p = other_p - # stress = "" - # while other_p and IPA.is_stress(other_p[0]): - # stress = other_p[0] - # other_p = other_p[1:] - - # if not other_p: - # continue - - # if other_p in self._phoneme_to_id: - # word_p.append(original_p) - # continue - - # assert self._voice_phonemes is not None - # guessed = guess_phonemes( - # other_p, to_phonemes=self._voice_phonemes - # ) - # if guessed.phonemes: - # word_p.extend([p.text for p in guessed.phonemes]) - - # if word_p: - # sent_phonemes.append(word_p) - - # result.sentence_phonemes.append(sent_phonemes) - - # sent_phoneme_ids = phonemes2ids.phonemes2ids( - # word_phonemes=sent_phonemes, - # phoneme_to_id=self._phoneme_to_id, - # pad=self._config.phonemes.pad, - # bos=self._config.phonemes.bos, - # eos=self._config.phonemes.eos, - # auto_bos_eos=self._config.phonemes.auto_bos_eos, - # blank=self._config.phonemes.blank, - # blank_word=self._config.phonemes.blank_word, - # blank_between=self._config.phonemes.blank_between, - # blank_at_start=self._config.phonemes.blank_at_start, - # blank_at_end=self._config.phonemes.blank_at_end, - # simple_punctuation=self._config.phonemes.simple_punctuation, - # punctuation_map=self._config.phonemes.punctuation_map, - # separate=self._config.phonemes.separate, - # separate_graphemes=self._config.phonemes.separate_graphemes, - # separate_tones=self._config.phonemes.separate_tones, - # tone_before=self._config.phonemes.tone_before, - # phoneme_map=self._phoneme_map or self._config.phonemes.phoneme_map, - # fail_on_missing=False, - # ) - - # result.sentence_phoneme_ids.append(sent_phonemes) - - # _LOGGER.debug("%s %s %s", sentence.text, sent_phonemes, sent_phoneme_ids) - - # # Create model inputs - # text_array = np.expand_dims(np.array(sent_phoneme_ids, dtype=np.int64), 0) - # text_lengths_array = np.array([text_array.shape[1]], dtype=np.int64) - # scales_array = np.array( - # [noise_scale, length_scale, noise_w], dtype=np.float32 - # ) - - # inputs = { - # "input": text_array, - # "input_lengths": text_lengths_array, - # "scales": scales_array, - # } - - # if self._config.is_multispeaker: - # speaker_id_array = np.array([speaker_id], dtype=np.int64) - # inputs["sid"] = speaker_id_array - - # # Infer audio from phonemes - # start_time = time.perf_counter() - # audio = self._onnx_model.run(None, inputs)[0].squeeze() - # audio = audio_float_to_int16(audio) - # end_time = time.perf_counter() - - # # Compute real-time factor - # audio_duration_sec = audio.shape[-1] / self._config.audio.sample_rate - # infer_sec = end_time - start_time - # real_time_factor = ( - # infer_sec / audio_duration_sec if audio_duration_sec > 0 else 0.0 - # ) - - # _LOGGER.debug("RTF: %s", real_time_factor) - - # audio_arrays.append(audio) - - # # Write to WAV and return bytes - # with io.BytesIO() as wav_file: - # write_wav( - # wav_file, self._config.audio.sample_rate, np.concatenate(audio_arrays), - # ) - - # result.wav_bytes = wav_file.getvalue() - - # return result diff --git a/mimic3-tts/mimic3_tts/voice.py b/mimic3-tts/mimic3_tts/voice.py index a8b8bca..e22adfb 100644 --- a/mimic3-tts/mimic3_tts/voice.py +++ b/mimic3-tts/mimic3_tts/voice.py @@ -117,15 +117,29 @@ class Mimic3Voice(metaclass=ABCMeta): speaker: typing.Optional[ typing.Union[SPEAKER_NAME_TYPE, SPEAKER_ID_TYPE] ] = None, - length_scale: float = 1.0, - noise_scale: float = 0.333, - noise_w: float = 1.0, + length_scale: typing.Optional[float] = None, + noise_scale: typing.Optional[float] = None, + noise_w: typing.Optional[float] = None, ) -> np.ndarray: + if length_scale is None: + length_scale = self.config.inference.length_scale + + if noise_scale is None: + noise_scale = self.config.inference.noise_scale + + if noise_w is None: + noise_w = self.config.inference.noise_w + # Create model inputs text_array = np.expand_dims(np.array(phoneme_ids, dtype=np.int64), 0) text_lengths_array = np.array([text_array.shape[1]], dtype=np.int64) scales_array = np.array( - [noise_scale, length_scale, noise_w,], dtype=np.float32, + [ + noise_scale, + length_scale, + noise_w, + ], + dtype=np.float32, ) # TODO: Use settings from voice config @@ -135,8 +149,8 @@ class Mimic3Voice(metaclass=ABCMeta): "scales": scales_array, } + speaker_id = 0 if self.config.is_multispeaker: - speaker_id = 0 if isinstance(speaker, SPEAKER_NAME_TYPE): if self.speaker_map: maybe_speaker_id = self.speaker_map.get(speaker) @@ -158,6 +172,14 @@ class Mimic3Voice(metaclass=ABCMeta): speaker_id_array = np.array([speaker_id], dtype=np.int64) inputs["sid"] = speaker_id_array + _LOGGER.debug( + "TTS settings: speaker-id=%s, length-scale=%s, noise-scale=%s, noise-w=%s", + speaker_id, + length_scale, + noise_scale, + noise_w, + ) + # Infer audio from phonemes start_time = time.perf_counter() audio = self.onnx_model.run(None, inputs)[0].squeeze() diff --git a/opentts-abc/opentts_abc/ssml.py b/opentts-abc/opentts_abc/ssml.py index e993675..e96bea4 100644 --- a/opentts-abc/opentts_abc/ssml.py +++ b/opentts-abc/opentts_abc/ssml.py @@ -111,6 +111,8 @@ class SSMLSpeaker: pass elif end_tag in {"metadata", "meta"}: self.handle_end_metadata() + elif end_tag == "speak": + yield from self.handle_end_speak() else: LOG.debug("Ignoring end tag: %s", end_tag) else: @@ -278,6 +280,12 @@ class SSMLSpeaker: yield from self.tts.end_utterance() + def handle_end_speak(self) -> typing.Iterable[BaseResult]: + LOG.debug("end speak") + assert self.state in {ParsingState.DEFAULT}, self.state + + yield from self.tts.end_utterance() + def handle_begin_voice(self, elem: etree.Element): LOG.debug("begin voice") voice_name = attrib_no_namespace(elem, "name")