Add manual minor/major breaks for eSpeak voices

mimic3-http/mimic3_http/client.py

@@ -15,11 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 import argparse
-import subprocess
-import shlex
-import shutil
 import logging
 import os
+import shlex
+import shutil
+import subprocess
 import sys
 import tempfile
 import typing
@@ -118,16 +118,24 @@ def get_args() -> argparse.Namespace:
         help="URL of mimic3 HTTP server (default: http://localhost:59125/api/tts)",
     )
     parser.add_argument(
-        "--voice", "-v", help="Name of voice (expected in <voices-dir>/<language>)",
+        "--voice",
+        "-v",
+        help="Name of voice (expected in <voices-dir>/<language>)",
     )
     parser.add_argument(
-        "--output", "-o", help="Path to write WAV file (default: play audio)",
+        "--output",
+        "-o",
+        help="Path to write WAV file (default: play audio)",
     )
     parser.add_argument(
-        "--stdout", action="store_true", help="Write WAV data to stdout",
+        "--stdout",
+        action="store_true",
+        help="Write WAV data to stdout",
     )
     parser.add_argument(
-        "--noise-scale", type=float, help="Noise scale [0-1], default is 0.667",
+        "--noise-scale",
+        type=float,
+        help="Noise scale [0-1], default is 0.667",
     )
     parser.add_argument(
         "--length-scale",
@@ -135,7 +143,9 @@ def get_args() -> argparse.Namespace:
         help="Length scale (1.0 is default speed, 0.5 is 2x faster)",
     )
     parser.add_argument(
-        "--noise-w", type=float, help="Variation in cadence [0-1], default is 0.8",
+        "--noise-w",
+        type=float,
+        help="Variation in cadence [0-1], default is 0.8",
     )
     parser.add_argument(
         "--play-program",

mimic3-http/mypy.ini

@@ -8,3 +8,6 @@ ignore_missing_imports = True
 
 [mypy-setuptools.*]
 ignore_missing_imports = True
+
+[mypy-swagger_ui.*]
+ignore_missing_imports = True

mimic3-tts/mimic3_tts/config.py

@@ -29,6 +29,8 @@ from phonemes2ids import BlankBetween
 
 @dataclass
 class AudioConfig(DataClassJsonMixin):
+    """Audio input/output details"""
+
     filter_length: int = 1024
     hop_length: int = 256
     win_length: int = 1024
@@ -109,6 +111,8 @@ class AudioConfig(DataClassJsonMixin):
 
 @dataclass
 class ModelConfig(DataClassJsonMixin):
+    """TTS model hyperparameters"""
+
     num_symbols: int = 0
     n_speakers: int = 1
 
@@ -141,6 +145,8 @@ class ModelConfig(DataClassJsonMixin):
 
 @dataclass
 class PhonemesConfig(DataClassJsonMixin):
+    """Phonemes to ids configuration"""
+
     phoneme_separator: str = " "
     """Separator between individual phonemes in CSV input"""
 
@@ -185,21 +191,30 @@ class PhonemesConfig(DataClassJsonMixin):
 
 
 class Phonemizer(str, Enum):
+    """Method used to convert text to phonemes"""
+
     SYMBOLS = "symbols"
     GRUUT = "gruut"
     ESPEAK = "espeak"
 
 
 class Aligner(str, Enum):
+    """Text/audio aligner"""
+
     KALDI_ALIGN = "kaldi_align"
+    """https://github.com/rhasspy/kaldi-align"""
 
 
 class TextCasing(str, Enum):
+    """Casing method applied to text"""
+
     LOWER = "lower"
     UPPER = "upper"
 
 
 class MetadataFormat(str, Enum):
+    """Format of training metadata"""
+
     TEXT = "text"
     PHONEMES = "phonemes"
     PHONEME_IDS = "ids"
@@ -207,6 +222,8 @@ class MetadataFormat(str, Enum):
 
 @dataclass
 class DatasetConfig:
+    """Training dataset configuration"""
+
     name: str
     metadata_format: MetadataFormat = MetadataFormat.TEXT
     multispeaker: bool = False
@@ -228,19 +245,28 @@ class DatasetConfig:
 
 @dataclass
 class AlignerConfig:
+    """Text/audio alignment configuration"""
+
     aligner: typing.Optional[Aligner] = None
     casing: typing.Optional[TextCasing] = None
 
 
 @dataclass
 class InferenceConfig:
+    """Inference configuration"""
+
     length_scale: float = 1.0
     noise_scale: float = 0.667
     noise_w: float = 0.8
 
+    minor_break_ms: typing.Optional[int] = None
+    major_break_ms: typing.Optional[int] = None
+
 
 @dataclass
 class TrainingConfig(DataClassJsonMixin):
+    """Master configuration for training"""
+
     seed: int = 1234
     epochs: int = 10000
     learning_rate: float = 2e-4

mimic3-tts/mimic3_tts/tts.py

@@ -44,7 +44,7 @@ from .const import (
     DEFAULT_VOICES_URL_FORMAT,
 )
 from .download import VoiceFile, download_voice
-from .voice import SPEAKER_TYPE, Mimic3Voice
+from .voice import SPEAKER_TYPE, BreakType, Mimic3Voice
 
 _DIR = Path(__file__).parent
 
@@ -293,15 +293,29 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
     def speak_text(self, text: str, text_language: typing.Optional[str] = None):
         voice = self._get_or_load_voice(self.voice)
 
-        for sent_phonemes in voice.text_to_phonemes(text, text_language=text_language):
+        minor_break_ms = voice.config.inference.major_break_ms
+        major_break_ms = voice.config.inference.major_break_ms
+
+        for sent_phonemes, break_type in voice.text_to_phonemes(
+            text, text_language=text_language
+        ):
+            # Utterances have start/end meta phonemes (usually ^ and $)
+            is_utterance = break_type != BreakType.NONE
+
             self._results.append(
                 Mimic3Phonemes(
                     current_settings=deepcopy(self.settings),
                     phonemes=sent_phonemes,
-                    is_utterance=False,
+                    is_utterance=is_utterance,
                 )
             )
 
+            # Add silence if using manual break intervals
+            if (break_type == BreakType.MAJOR) and (major_break_ms is not None):
+                self.add_break(major_break_ms)
+            elif (break_type == BreakType.MINOR) and (minor_break_ms is not None):
+                self.add_break(minor_break_ms)
+
     # pylint: disable=arguments-differ
     def speak_tokens(
         self,

mimic3-tts/mimic3_tts/voice.py

@@ -19,6 +19,7 @@ import platform
 import time
 import typing
 from abc import ABCMeta, abstractmethod
+from enum import Enum
 from pathlib import Path
 from xml.sax.saxutils import escape as xmlescape
 
@@ -34,10 +35,19 @@ from mimic3_tts.utils import audio_float_to_int16
 
 # -----------------------------------------------------------------------------
 
+
+class BreakType(str, Enum):
+    NONE = "none"
+    MINOR = "minor"
+    MAJOR = "major"
+    UTTERANCE = "utterance"
+
+
 PHONEME_TYPE = str
 PHONEME_ID_TYPE = int
 WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]]
 PHONEME_MAP_TYPE = typing.Dict[PHONEME_TYPE, typing.List[PHONEME_TYPE]]
+TEXT_TO_PHONEMES_TYPE = typing.Iterable[typing.Tuple[WORD_PHONEMES_TYPE, BreakType]]
 
 SPEAKER_NAME_TYPE = str
 SPEAKER_ID_TYPE = int
@@ -48,6 +58,7 @@ DEFAULT_LANGUAGE = "en_US"
 
 _LOGGER = logging.getLogger(__name__)
 
+
 # -----------------------------------------------------------------------------
 
 
@@ -71,7 +82,7 @@ class Mimic3Voice(metaclass=ABCMeta):
     @abstractmethod
     def text_to_phonemes(
         self, text: str, text_language: typing.Optional[str] = None
-    ) -> typing.Iterable[WORD_PHONEMES_TYPE]:
+    ) -> TEXT_TO_PHONEMES_TYPE:
         """Convert text into phonemes"""
 
     def word_to_phonemes(
@@ -82,7 +93,7 @@ class Mimic3Voice(metaclass=ABCMeta):
     ) -> typing.List[PHONEME_TYPE]:
         """Convert a single word (with optional role) into phonemes"""
         word_phonemes = []
-        for sent_phonemes in self.text_to_phonemes(
+        for sent_phonemes, _break_type in self.text_to_phonemes(
             word_text, text_language=text_language
         ):
             for sent_word_phonemes in sent_phonemes:
@@ -99,7 +110,9 @@ class Mimic3Voice(metaclass=ABCMeta):
     ) -> WORD_PHONEMES_TYPE:
         """Speak a word or phrase with a specific interpretation/format"""
         word_phonemes = []
-        for sent_phonemes in self.text_to_phonemes(text, text_language=text_language):
+        for sent_phonemes, _break_type in self.text_to_phonemes(
+            text, text_language=text_language
+        ):
             word_phonemes.extend(sent_phonemes)
 
         return word_phonemes
@@ -318,12 +331,12 @@ class GruutVoice(Mimic3Voice):
 
     def text_to_phonemes(
         self, text: str, text_language: typing.Optional[str] = None
-    ) -> typing.Iterable[WORD_PHONEMES_TYPE]:
+    ) -> TEXT_TO_PHONEMES_TYPE:
         text_language = text_language or self.config.text_language or DEFAULT_LANGUAGE
         for sentence in gruut.sentences(text, lang=text_language):
             sent_phonemes = [w.phonemes for w in sentence if w.phonemes]
             if sent_phonemes:
-                yield sent_phonemes
+                yield sent_phonemes, BreakType.NONE
 
     def word_to_phonemes(
         self,
@@ -389,7 +402,7 @@ class EspeakVoice(Mimic3Voice):
 
     def text_to_phonemes(
         self, text: str, text_language: typing.Optional[str] = None
-    ) -> typing.Iterable[WORD_PHONEMES_TYPE]:
+    ) -> TEXT_TO_PHONEMES_TYPE:
         phoneme_separator = ""
         word_separator = self.config.phonemes.word_separator
 
@@ -406,11 +419,31 @@ class EspeakVoice(Mimic3Voice):
             punctuation_separator=phoneme_separator,
         )
 
-        word_phonemes = [
+        all_word_phonemes = [
             list(IPA.graphemes(wp_str)) for wp_str in phoneme_str.split(word_separator)
         ]
 
-        yield word_phonemes
+        minor_break = self.config.phonemes.minor_break
+        major_break = self.config.phonemes.major_break
+
+        if minor_break or major_break:
+            # Split on breaks
+            sent_phonemes = []
+            for word_phonemes in all_word_phonemes:
+                sent_phonemes.append(word_phonemes)
+
+                if minor_break and (word_phonemes[-1] == minor_break):
+                    yield sent_phonemes, BreakType.MINOR
+                    sent_phonemes = []
+                elif major_break and (word_phonemes[-1] == major_break):
+                    yield sent_phonemes, BreakType.MAJOR
+                    sent_phonemes = []
+
+            if sent_phonemes:
+                yield sent_phonemes, BreakType.MAJOR
+        else:
+            # No split
+            yield all_word_phonemes, BreakType.UTTERANCE
 
     def word_to_phonemes(
         self,
@@ -486,9 +519,9 @@ class SymbolsVoice(Mimic3Voice):
 
     def text_to_phonemes(
         self, text: str, text_language: typing.Optional[str] = None
-    ) -> typing.Iterable[WORD_PHONEMES_TYPE]:
+    ) -> TEXT_TO_PHONEMES_TYPE:
         word_separator = self.config.phonemes.word_separator
         word_phonemes = [
             list(IPA.graphemes(wp_str)) for wp_str in text.split(word_separator)
         ]
-        yield word_phonemes
+        yield word_phonemes, BreakType.NONE

mimic3-tts/mimic3_tts/voices.json

@@ -63,6 +63,36 @@
         "speakers": [],
         "properties": {}
     },
+    "en_UK/apope_low": {
+        "files": {
+            "LICENSE": {
+                "size_bytes": 46,
+                "sha256_sum": "2fb2b744e6d96a2e3137d58121670b005718c736d5dde1832c99ff9073f64c00"
+            },
+            "README.md": {
+                "size_bytes": 155,
+                "sha256_sum": "2e6c39454c35910c6b48518523429ec2e4f27ae8eb763e311877a017db23d4da"
+            },
+            "config.json": {
+                "size_bytes": 3434,
+                "sha256_sum": "1fdaa1124e02cc177eb776fbc6e08c838b56bd2e86c82d8d7fe434d9337806b0"
+            },
+            "generator.onnx": {
+                "size_bytes": 62792219,
+                "sha256_sum": "0b5a323500ebd022351db12da2b3aab8cdd47d0826d173e780a58b93604618c9"
+            },
+            "phoneme_map.txt": {
+                "size_bytes": 15,
+                "sha256_sum": "4003f421fc91ed1d5a343442659db6cf9d58bd1c6d8d771abc1999cc24d7694d"
+            },
+            "phonemes.txt": {
+                "size_bytes": 263,
+                "sha256_sum": "8f9c3e6ced14d7fc5426e4e1bc7f7cc1037a20a645ca34110abcb76148fa8bfd"
+            }
+        },
+        "speakers": [],
+        "properties": {}
+    },
     "en_US/cmu-arctic_low": {
         "files": {
             "LICENSE": {
@@ -502,6 +532,40 @@
         "speakers": [],
         "properties": {}
     },
+    "nl/nathalie_low": {
+        "files": {
+            "LICENSE": {
+                "size_bytes": 7049,
+                "sha256_sum": "7179683e8000e6bdc9bbc60d85edf0a4ac8e76f951857f54fcb775d5886f1309"
+            },
+            "README.md": {
+                "size_bytes": 180,
+                "sha256_sum": "15e74dbbdadc8bf7c3a84b1b130f6e5df5f37ffe82880703b27d92e11cff1af8"
+            },
+            "SOURCE": {
+                "size_bytes": 50,
+                "sha256_sum": "6422e7d891e0d77db7a7cd9643bc8cff4c26f47d652b83b651a0179682ce65c7"
+            },
+            "config.json": {
+                "size_bytes": 3604,
+                "sha256_sum": "e8e1acf414067657bd377965eb822c51f55483e2ad9a498398e473b0eae4c137"
+            },
+            "generator.onnx": {
+                "size_bytes": 62800663,
+                "sha256_sum": "1707e33a94538529009c28d0da2a99fe81aa15bd7bb4c2252b3ace0ab704fa71"
+            },
+            "phoneme_map.txt": {
+                "size_bytes": 21,
+                "sha256_sum": "18b9c5fe46201cd5b0b20e88dc78c64ef3bcaaf87a428386420757f2d0bf7bb3"
+            },
+            "phonemes.txt": {
+                "size_bytes": 336,
+                "sha256_sum": "355389fee04f97557232cdde7fb8d4cf03ae2aabd7b0b26ed5978ebbf6575dd4"
+            }
+        },
+        "speakers": [],
+        "properties": {}
+    },
     "nl/rdh_low": {
         "files": {
             "LICENSE": {