From 9fd10f59491da743ff10ef16dee7890ffc106ff7 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Tue, 5 Apr 2022 10:50:47 -0400 Subject: [PATCH] Add manual minor/major breaks for eSpeak voices --- mimic3-http/mimic3_http/client.py | 26 +++++++++---- mimic3-http/mypy.ini | 3 ++ mimic3-tts/mimic3_tts/config.py | 26 +++++++++++++ mimic3-tts/mimic3_tts/tts.py | 20 ++++++++-- mimic3-tts/mimic3_tts/voice.py | 53 ++++++++++++++++++++----- mimic3-tts/mimic3_tts/voices.json | 64 +++++++++++++++++++++++++++++++ 6 files changed, 171 insertions(+), 21 deletions(-) diff --git a/mimic3-http/mimic3_http/client.py b/mimic3-http/mimic3_http/client.py index 0da0f6b..8336710 100644 --- a/mimic3-http/mimic3_http/client.py +++ b/mimic3-http/mimic3_http/client.py @@ -15,11 +15,11 @@ # along with this program. If not, see . # import argparse -import subprocess -import shlex -import shutil import logging import os +import shlex +import shutil +import subprocess import sys import tempfile import typing @@ -118,16 +118,24 @@ def get_args() -> argparse.Namespace: help="URL of mimic3 HTTP server (default: http://localhost:59125/api/tts)", ) parser.add_argument( - "--voice", "-v", help="Name of voice (expected in /)", + "--voice", + "-v", + help="Name of voice (expected in /)", ) parser.add_argument( - "--output", "-o", help="Path to write WAV file (default: play audio)", + "--output", + "-o", + help="Path to write WAV file (default: play audio)", ) parser.add_argument( - "--stdout", action="store_true", help="Write WAV data to stdout", + "--stdout", + action="store_true", + help="Write WAV data to stdout", ) parser.add_argument( - "--noise-scale", type=float, help="Noise scale [0-1], default is 0.667", + "--noise-scale", + type=float, + help="Noise scale [0-1], default is 0.667", ) parser.add_argument( "--length-scale", @@ -135,7 +143,9 @@ def get_args() -> argparse.Namespace: help="Length scale (1.0 is default speed, 0.5 is 2x faster)", ) parser.add_argument( - "--noise-w", type=float, help="Variation in cadence [0-1], default is 0.8", + "--noise-w", + type=float, + help="Variation in cadence [0-1], default is 0.8", ) parser.add_argument( "--play-program", diff --git a/mimic3-http/mypy.ini b/mimic3-http/mypy.ini index c1e912c..7a7332e 100644 --- a/mimic3-http/mypy.ini +++ b/mimic3-http/mypy.ini @@ -8,3 +8,6 @@ ignore_missing_imports = True [mypy-setuptools.*] ignore_missing_imports = True + +[mypy-swagger_ui.*] +ignore_missing_imports = True diff --git a/mimic3-tts/mimic3_tts/config.py b/mimic3-tts/mimic3_tts/config.py index 945a282..b2c2d5b 100644 --- a/mimic3-tts/mimic3_tts/config.py +++ b/mimic3-tts/mimic3_tts/config.py @@ -29,6 +29,8 @@ from phonemes2ids import BlankBetween @dataclass class AudioConfig(DataClassJsonMixin): + """Audio input/output details""" + filter_length: int = 1024 hop_length: int = 256 win_length: int = 1024 @@ -109,6 +111,8 @@ class AudioConfig(DataClassJsonMixin): @dataclass class ModelConfig(DataClassJsonMixin): + """TTS model hyperparameters""" + num_symbols: int = 0 n_speakers: int = 1 @@ -141,6 +145,8 @@ class ModelConfig(DataClassJsonMixin): @dataclass class PhonemesConfig(DataClassJsonMixin): + """Phonemes to ids configuration""" + phoneme_separator: str = " " """Separator between individual phonemes in CSV input""" @@ -185,21 +191,30 @@ class PhonemesConfig(DataClassJsonMixin): class Phonemizer(str, Enum): + """Method used to convert text to phonemes""" + SYMBOLS = "symbols" GRUUT = "gruut" ESPEAK = "espeak" class Aligner(str, Enum): + """Text/audio aligner""" + KALDI_ALIGN = "kaldi_align" + """https://github.com/rhasspy/kaldi-align""" class TextCasing(str, Enum): + """Casing method applied to text""" + LOWER = "lower" UPPER = "upper" class MetadataFormat(str, Enum): + """Format of training metadata""" + TEXT = "text" PHONEMES = "phonemes" PHONEME_IDS = "ids" @@ -207,6 +222,8 @@ class MetadataFormat(str, Enum): @dataclass class DatasetConfig: + """Training dataset configuration""" + name: str metadata_format: MetadataFormat = MetadataFormat.TEXT multispeaker: bool = False @@ -228,19 +245,28 @@ class DatasetConfig: @dataclass class AlignerConfig: + """Text/audio alignment configuration""" + aligner: typing.Optional[Aligner] = None casing: typing.Optional[TextCasing] = None @dataclass class InferenceConfig: + """Inference configuration""" + length_scale: float = 1.0 noise_scale: float = 0.667 noise_w: float = 0.8 + minor_break_ms: typing.Optional[int] = None + major_break_ms: typing.Optional[int] = None + @dataclass class TrainingConfig(DataClassJsonMixin): + """Master configuration for training""" + seed: int = 1234 epochs: int = 10000 learning_rate: float = 2e-4 diff --git a/mimic3-tts/mimic3_tts/tts.py b/mimic3-tts/mimic3_tts/tts.py index 6550694..9b0f197 100644 --- a/mimic3-tts/mimic3_tts/tts.py +++ b/mimic3-tts/mimic3_tts/tts.py @@ -44,7 +44,7 @@ from .const import ( DEFAULT_VOICES_URL_FORMAT, ) from .download import VoiceFile, download_voice -from .voice import SPEAKER_TYPE, Mimic3Voice +from .voice import SPEAKER_TYPE, BreakType, Mimic3Voice _DIR = Path(__file__).parent @@ -293,15 +293,29 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): def speak_text(self, text: str, text_language: typing.Optional[str] = None): voice = self._get_or_load_voice(self.voice) - for sent_phonemes in voice.text_to_phonemes(text, text_language=text_language): + minor_break_ms = voice.config.inference.major_break_ms + major_break_ms = voice.config.inference.major_break_ms + + for sent_phonemes, break_type in voice.text_to_phonemes( + text, text_language=text_language + ): + # Utterances have start/end meta phonemes (usually ^ and $) + is_utterance = break_type != BreakType.NONE + self._results.append( Mimic3Phonemes( current_settings=deepcopy(self.settings), phonemes=sent_phonemes, - is_utterance=False, + is_utterance=is_utterance, ) ) + # Add silence if using manual break intervals + if (break_type == BreakType.MAJOR) and (major_break_ms is not None): + self.add_break(major_break_ms) + elif (break_type == BreakType.MINOR) and (minor_break_ms is not None): + self.add_break(minor_break_ms) + # pylint: disable=arguments-differ def speak_tokens( self, diff --git a/mimic3-tts/mimic3_tts/voice.py b/mimic3-tts/mimic3_tts/voice.py index df85146..d24db86 100644 --- a/mimic3-tts/mimic3_tts/voice.py +++ b/mimic3-tts/mimic3_tts/voice.py @@ -19,6 +19,7 @@ import platform import time import typing from abc import ABCMeta, abstractmethod +from enum import Enum from pathlib import Path from xml.sax.saxutils import escape as xmlescape @@ -34,10 +35,19 @@ from mimic3_tts.utils import audio_float_to_int16 # ----------------------------------------------------------------------------- + +class BreakType(str, Enum): + NONE = "none" + MINOR = "minor" + MAJOR = "major" + UTTERANCE = "utterance" + + PHONEME_TYPE = str PHONEME_ID_TYPE = int WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]] PHONEME_MAP_TYPE = typing.Dict[PHONEME_TYPE, typing.List[PHONEME_TYPE]] +TEXT_TO_PHONEMES_TYPE = typing.Iterable[typing.Tuple[WORD_PHONEMES_TYPE, BreakType]] SPEAKER_NAME_TYPE = str SPEAKER_ID_TYPE = int @@ -48,6 +58,7 @@ DEFAULT_LANGUAGE = "en_US" _LOGGER = logging.getLogger(__name__) + # ----------------------------------------------------------------------------- @@ -71,7 +82,7 @@ class Mimic3Voice(metaclass=ABCMeta): @abstractmethod def text_to_phonemes( self, text: str, text_language: typing.Optional[str] = None - ) -> typing.Iterable[WORD_PHONEMES_TYPE]: + ) -> TEXT_TO_PHONEMES_TYPE: """Convert text into phonemes""" def word_to_phonemes( @@ -82,7 +93,7 @@ class Mimic3Voice(metaclass=ABCMeta): ) -> typing.List[PHONEME_TYPE]: """Convert a single word (with optional role) into phonemes""" word_phonemes = [] - for sent_phonemes in self.text_to_phonemes( + for sent_phonemes, _break_type in self.text_to_phonemes( word_text, text_language=text_language ): for sent_word_phonemes in sent_phonemes: @@ -99,7 +110,9 @@ class Mimic3Voice(metaclass=ABCMeta): ) -> WORD_PHONEMES_TYPE: """Speak a word or phrase with a specific interpretation/format""" word_phonemes = [] - for sent_phonemes in self.text_to_phonemes(text, text_language=text_language): + for sent_phonemes, _break_type in self.text_to_phonemes( + text, text_language=text_language + ): word_phonemes.extend(sent_phonemes) return word_phonemes @@ -318,12 +331,12 @@ class GruutVoice(Mimic3Voice): def text_to_phonemes( self, text: str, text_language: typing.Optional[str] = None - ) -> typing.Iterable[WORD_PHONEMES_TYPE]: + ) -> TEXT_TO_PHONEMES_TYPE: text_language = text_language or self.config.text_language or DEFAULT_LANGUAGE for sentence in gruut.sentences(text, lang=text_language): sent_phonemes = [w.phonemes for w in sentence if w.phonemes] if sent_phonemes: - yield sent_phonemes + yield sent_phonemes, BreakType.NONE def word_to_phonemes( self, @@ -389,7 +402,7 @@ class EspeakVoice(Mimic3Voice): def text_to_phonemes( self, text: str, text_language: typing.Optional[str] = None - ) -> typing.Iterable[WORD_PHONEMES_TYPE]: + ) -> TEXT_TO_PHONEMES_TYPE: phoneme_separator = "" word_separator = self.config.phonemes.word_separator @@ -406,11 +419,31 @@ class EspeakVoice(Mimic3Voice): punctuation_separator=phoneme_separator, ) - word_phonemes = [ + all_word_phonemes = [ list(IPA.graphemes(wp_str)) for wp_str in phoneme_str.split(word_separator) ] - yield word_phonemes + minor_break = self.config.phonemes.minor_break + major_break = self.config.phonemes.major_break + + if minor_break or major_break: + # Split on breaks + sent_phonemes = [] + for word_phonemes in all_word_phonemes: + sent_phonemes.append(word_phonemes) + + if minor_break and (word_phonemes[-1] == minor_break): + yield sent_phonemes, BreakType.MINOR + sent_phonemes = [] + elif major_break and (word_phonemes[-1] == major_break): + yield sent_phonemes, BreakType.MAJOR + sent_phonemes = [] + + if sent_phonemes: + yield sent_phonemes, BreakType.MAJOR + else: + # No split + yield all_word_phonemes, BreakType.UTTERANCE def word_to_phonemes( self, @@ -486,9 +519,9 @@ class SymbolsVoice(Mimic3Voice): def text_to_phonemes( self, text: str, text_language: typing.Optional[str] = None - ) -> typing.Iterable[WORD_PHONEMES_TYPE]: + ) -> TEXT_TO_PHONEMES_TYPE: word_separator = self.config.phonemes.word_separator word_phonemes = [ list(IPA.graphemes(wp_str)) for wp_str in text.split(word_separator) ] - yield word_phonemes + yield word_phonemes, BreakType.NONE diff --git a/mimic3-tts/mimic3_tts/voices.json b/mimic3-tts/mimic3_tts/voices.json index eb28112..82b3c60 100644 --- a/mimic3-tts/mimic3_tts/voices.json +++ b/mimic3-tts/mimic3_tts/voices.json @@ -63,6 +63,36 @@ "speakers": [], "properties": {} }, + "en_UK/apope_low": { + "files": { + "LICENSE": { + "size_bytes": 46, + "sha256_sum": "2fb2b744e6d96a2e3137d58121670b005718c736d5dde1832c99ff9073f64c00" + }, + "README.md": { + "size_bytes": 155, + "sha256_sum": "2e6c39454c35910c6b48518523429ec2e4f27ae8eb763e311877a017db23d4da" + }, + "config.json": { + "size_bytes": 3434, + "sha256_sum": "1fdaa1124e02cc177eb776fbc6e08c838b56bd2e86c82d8d7fe434d9337806b0" + }, + "generator.onnx": { + "size_bytes": 62792219, + "sha256_sum": "0b5a323500ebd022351db12da2b3aab8cdd47d0826d173e780a58b93604618c9" + }, + "phoneme_map.txt": { + "size_bytes": 15, + "sha256_sum": "4003f421fc91ed1d5a343442659db6cf9d58bd1c6d8d771abc1999cc24d7694d" + }, + "phonemes.txt": { + "size_bytes": 263, + "sha256_sum": "8f9c3e6ced14d7fc5426e4e1bc7f7cc1037a20a645ca34110abcb76148fa8bfd" + } + }, + "speakers": [], + "properties": {} + }, "en_US/cmu-arctic_low": { "files": { "LICENSE": { @@ -502,6 +532,40 @@ "speakers": [], "properties": {} }, + "nl/nathalie_low": { + "files": { + "LICENSE": { + "size_bytes": 7049, + "sha256_sum": "7179683e8000e6bdc9bbc60d85edf0a4ac8e76f951857f54fcb775d5886f1309" + }, + "README.md": { + "size_bytes": 180, + "sha256_sum": "15e74dbbdadc8bf7c3a84b1b130f6e5df5f37ffe82880703b27d92e11cff1af8" + }, + "SOURCE": { + "size_bytes": 50, + "sha256_sum": "6422e7d891e0d77db7a7cd9643bc8cff4c26f47d652b83b651a0179682ce65c7" + }, + "config.json": { + "size_bytes": 3604, + "sha256_sum": "e8e1acf414067657bd377965eb822c51f55483e2ad9a498398e473b0eae4c137" + }, + "generator.onnx": { + "size_bytes": 62800663, + "sha256_sum": "1707e33a94538529009c28d0da2a99fe81aa15bd7bb4c2252b3ace0ab704fa71" + }, + "phoneme_map.txt": { + "size_bytes": 21, + "sha256_sum": "18b9c5fe46201cd5b0b20e88dc78c64ef3bcaaf87a428386420757f2d0bf7bb3" + }, + "phonemes.txt": { + "size_bytes": 336, + "sha256_sum": "355389fee04f97557232cdde7fb8d4cf03ae2aabd7b0b26ed5978ebbf6575dd4" + } + }, + "speakers": [], + "properties": {} + }, "nl/rdh_low": { "files": { "LICENSE": {