Cleanup in mimic3_tts

mimic3-tts/mimic3_tts/__init__.py

@@ -1,4 +1,9 @@
+from pathlib import Path
+
 from opentts_abc import AudioResult, MarkResult
 from opentts_abc.ssml import SSMLSpeaker
 
+from ._resources import __version__
 from .tts import Mimic3Settings, Mimic3TextToSpeechSystem
+
+__author__ = "Michael Hansen"

mimic3-tts/mimic3_tts/__main__.py

@@ -2,16 +2,11 @@
 import logging
 import wave
 
-logging.basicConfig(level=logging.DEBUG)
-
 from opentts_abc.ssml import SSMLSpeaker
 
-from mimic3_tts.tts import (
+from .tts import AudioResult, MarkResult, Mimic3Settings, Mimic3TextToSpeechSystem
-    AudioResult,
+
-    MarkResult,
+logging.basicConfig(level=logging.DEBUG)
-    Mimic3Settings,
-    Mimic3TextToSpeechSystem,
-)
 
 settings = Mimic3Settings()
 tts = Mimic3TextToSpeechSystem(settings)
@@ -22,21 +17,21 @@ speaker = SSMLSpeaker(tts)
 # ssml = '<speak><s><w>Hello</w><w>World</w></s></speak>'
 # ssml = '<speak><s>Hello world</s></speak>'
 # ssml = '<speak><s><voice name="el_GR/rapunzelina_low"><say-as interpret-as="characters">12</say-as></voice></s></speak>'
-ssml = '''
+ssml = """
 <speak>
 <voice name="en_US/amy_low">
-  <s>
+  Today is a test.
-    Today is 1/2.
+  This is another test.
-    <break time="1s" />
-  </s>
 </voice>
 
 
 <voice name="es_ES/carlfm_low">
-  <s>Soy el <say-as interpret-as="number" format="ordinal">1</say-as>.</s>
+  <lang xml:lang="es_ES">
+    Soy el <say-as interpret-as="number" format="ordinal">1</say-as>.
+  </lang>
 </voice>
 </speak>
-'''
+"""
 
 
 wav_file: wave.Wave_write = wave.open("out.wav", "wb")

mimic3-tts/mimic3_tts/_resources.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+import os
+import typing
+from pathlib import Path
+
+try:
+    import importlib.resources
+
+    files = importlib.resources.files
+except (ImportError, AttributeError):
+    # Backport for Python < 3.9
+    import importlib_resources  # type: ignore
+
+    files = importlib_resources.files
+
+_PACKAGE = "mimic3_tts"
+_DIR = Path(typing.cast(os.PathLike, files(_PACKAGE)))
+
+__version__ = (_DIR / "VERSION").read_text(encoding="utf-8").strip()

mimic3-tts/mimic3_tts/config.py

@@ -272,18 +272,6 @@ class TrainingConfig(DataClassJsonMixin):
         """Save config as JSON to a file"""
         json.dump(self.to_dict(), config_file, indent=4)
 
-    def get_speaker_id(self, dataset_name: str, speaker_name: str) -> int:
-        if self.speaker_id_map is None:
-            self.speaker_id_map = {}
-
-        full_speaker_name = f"{dataset_name}_{speaker_name}"
-        speaker_id = self.speaker_id_map.get(full_speaker_name)
-        if speaker_id is None:
-            speaker_id = len(self.speaker_id_map)
-            self.speaker_id_map[full_speaker_name] = speaker_id
-
-        return speaker_id
-
     @staticmethod
     def load(config_file: typing.TextIO) -> "TrainingConfig":
         """Load config from a JSON file"""

mimic3-tts/mimic3_tts/tts.py

@@ -1,18 +1,13 @@
 #!/usr/bin/env python3
+"""Implementation of OpenTTS for Mimic 3"""
+import itertools
 import logging
-import time
 import typing
 from copy import deepcopy
 from dataclasses import dataclass, field
 from pathlib import Path
-from xml.sax.saxutils import escape as xmlescape
 
-import gruut
+from gruut_ipa import IPA
-import numpy as np
-import onnxruntime
-import phonemes2ids
-from gruut.const import LookupPhonemes, WordRole
-from gruut_ipa import IPA, Phoneme, guess_phonemes
 from opentts_abc import (
     AudioResult,
     BaseResult,
@@ -24,15 +19,16 @@ from opentts_abc import (
     Voice,
     Word,
 )
+from xdgenvpy import XDG
 
-from mimic3_tts.config import TrainingConfig
+from .config import TrainingConfig
-from mimic3_tts.voice import Mimic3Voice, SPEAKER_TYPE
+from .voice import SPEAKER_TYPE, Mimic3Voice
 
 _DIR = Path(__file__).parent
 
 _LOGGER = logging.getLogger(__name__)
 
-PHONEMES_LIST = typing.List[typing.List[str]]
+PHONEMES_LIST_TYPE = typing.List[typing.List[str]]
 
 DEFAULT_VOICE = "en_US/vctk_low"
 DEFAULT_LANGUAGE = "en_US"
@@ -43,25 +39,53 @@ DEFAULT_LANGUAGE = "en_US"
 
 @dataclass
 class Mimic3Settings:
+    """Settings for Mimic 3 text to speech system"""
+
     voice: typing.Optional[str] = None
+    """Default voice key"""
+
     language: typing.Optional[str] = None
+    """Default language (e.g., "en_US")"""
+
     voices_directories: typing.Optional[typing.Iterable[typing.Union[str, Path]]] = None
+    """Directories to search for voices (<lang>/<voice>)"""
+
     speaker: typing.Optional[SPEAKER_TYPE] = None
+    """Default speaker name or id"""
+
     length_scale: typing.Optional[float] = None
+    """Default length scale (use voice config if None)"""
+
     noise_scale: typing.Optional[float] = None
+    """Default noise scale (use voice config if None)"""
+
     noise_w: typing.Optional[float] = None
+    """Default noise W (use voice config if None)"""
+
     text_language: typing.Optional[str] = None
+    """Language of text (use voice language if None)"""
+
     sample_rate: int = 22050
+    """Sample rate of silence from add_break() in Hertz"""
 
 
 @dataclass
 class Mimic3Phonemes:
+    """Pending task to synthesize audio from phonemes with specific settings"""
+
     current_settings: Mimic3Settings
+    """Settings used to synthesize audio"""
+
     phonemes: typing.List[typing.List[str]] = field(default_factory=list)
+    """Phonemes for synthesis"""
+
     is_utterance: bool = True
+    """True if this is the end of a full utterance"""
 
 
 class VoiceNotFoundError(Exception):
+    """Raised if a voice cannot be found"""
+
     def __init__(self, voice: str):
         super().__init__(f"Voice not found: {voice}")
 
@@ -78,57 +102,37 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
         self._results: typing.List[typing.Union[BaseResult, Mimic3Phonemes]] = []
         self._loaded_voices: typing.Dict[str, Mimic3Voice] = {}
 
-    @property
-    def voice(self) -> str:
-        return self.settings.voice or DEFAULT_VOICE
-
-    @voice.setter
-    def voice(self, new_voice: str):
-        if new_voice != self.settings.voice:
-            # Clear speaker on voice change
-            self.speaker = None
-
-        self.settings.voice = new_voice
-
-        if "#" in self.settings.voice:
-            # Split
-            voice, speaker = self.settings.voice.split("#", maxsplit=1)
-            self.settings.voice = voice
-            self.speaker = speaker
-
-    @property
-    def speaker(self) -> typing.Optional[SPEAKER_TYPE]:
-        return self.settings.speaker
-
-    @speaker.setter
-    def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]):
-        self.settings.speaker = new_speaker
-
-    @property
-    def language(self) -> str:
-        return self.settings.language or DEFAULT_LANGUAGE
-
-    @language.setter
-    def language(self, new_language: str):
-        self.settings.language = new_language
-
     @staticmethod
     def get_default_voices_directories() -> typing.List[Path]:
-        return [_DIR.parent.parent / "voices"]
+        """Get list of directories to search for voices by default.
+
+        On Linux, this is typically:
+            - $HOME/.local/share/mimic3
+            - /usr/local/share/mimic3
+            - /usr/share/mimic3
+        """
+        data_dirs = [Path(d) / "mimic3" for d in XDG().XDG_DATA_DIRS.split(":")]
+        return [_DIR.parent.parent / "voices"] + data_dirs
 
     def get_voices(self) -> typing.Iterable[Voice]:
-        voices_dirs = (
+        """Returns an iterable of all available voices"""
-            self.settings.voices_directories
+        voices_dirs: typing.Iterable[
-            or Mimic3TextToSpeechSystem.get_default_voices_directories()
+            typing.Union[str, Path]
-        )
+        ] = Mimic3TextToSpeechSystem.get_default_voices_directories()
+
+        if self.settings.voices_directories is not None:
+            voices_dirs = itertools.chain(self.settings.voices_directories, voices_dirs)
 
         # voices/<language>/<voice>/
         for voices_dir in voices_dirs:
             voices_dir = Path(voices_dir)
 
             if not voices_dir.is_dir():
+                _LOGGER.debug("Skipping voice directory %s", voices_dir)
                 continue
 
+            _LOGGER.debug("Searching %s for voices", voices_dir)
+
             for lang_dir in voices_dir.iterdir():
                 if not lang_dir.is_dir():
                     continue
@@ -137,6 +141,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
                     if not voice_dir.is_dir():
                         continue
 
+                    _LOGGER.debug("Voice found in %s", voice_dir)
                     voice_lang = lang_dir.name
 
                     # Load config
@@ -176,9 +181,50 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
                         properties=properties,
                     )
 
+    def preload_voice(self, voice_key: str):
+        """Ensure voice is loaded in memory before synthesis"""
+        self._get_or_load_voice(voice_key)
+
+    # -------------------------------------------------------------------------
+
+    @property
+    def voice(self) -> str:
+        return self.settings.voice or DEFAULT_VOICE
+
+    @voice.setter
+    def voice(self, new_voice: str):
+        if new_voice != self.settings.voice:
+            # Clear speaker on voice change
+            self.speaker = None
+
+        self.settings.voice = new_voice
+
+        if "#" in self.settings.voice:
+            # Split
+            voice, speaker = self.settings.voice.split("#", maxsplit=1)
+            self.settings.voice = voice
+            self.speaker = speaker
+
+    @property
+    def speaker(self) -> typing.Optional[SPEAKER_TYPE]:
+        return self.settings.speaker
+
+    @speaker.setter
+    def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]):
+        self.settings.speaker = new_speaker
+
+    @property
+    def language(self) -> str:
+        return self.settings.language or DEFAULT_LANGUAGE
+
+    @language.setter
+    def language(self, new_language: str):
+        self.settings.language = new_language
+
     def begin_utterance(self):
         pass
 
+    # pylint: disable=arguments-differ
     def speak_text(self, text: str, text_language: typing.Optional[str] = None):
         voice = self._get_or_load_voice(self.voice)
 
@@ -187,44 +233,18 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
                 Mimic3Phonemes(
                     current_settings=deepcopy(self.settings),
                     phonemes=sent_phonemes,
+                    is_utterance=False,
                 )
             )
 
-    def _speak_sentence_phonemes(
+    # pylint: disable=arguments-differ
-        self,
-        sent_phonemes,
-        settings: typing.Optional[Mimic3Settings] = None,
-    ) -> AudioResult:
-        settings = settings or self.settings
-        voice = self._get_or_load_voice(settings.voice or self.voice)
-        sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes)
-
-        _LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids)
-
-        audio = voice.ids_to_audio(
-            sent_phoneme_ids,
-            speaker=self.speaker,
-            length_scale=settings.length_scale,
-            noise_scale=settings.noise_scale,
-            noise_w=settings.noise_w,
-        )
-
-        audio_bytes = audio.tobytes()
-        return AudioResult(
-            sample_rate_hz=voice.config.audio.sample_rate,
-            audio_bytes=audio_bytes,
-            # 16-bit mono
-            sample_width_bytes=2,
-            num_channels=1,
-        )
-
     def speak_tokens(
         self,
         tokens: typing.Iterable[BaseToken],
         text_language: typing.Optional[str] = None,
     ):
         voice = self._get_or_load_voice(self.voice)
-        token_phonemes: PHONEMES_LIST = []
+        token_phonemes: PHONEMES_LIST_TYPE = []
 
         for token in tokens:
             if isinstance(token, Word):
@@ -250,7 +270,9 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
         if token_phonemes:
             self._results.append(
                 Mimic3Phonemes(
-                    current_settings=deepcopy(self.settings), phonemes=token_phonemes
+                    current_settings=deepcopy(self.settings),
+                    phonemes=token_phonemes,
+                    is_utterance=False,
                 )
             )
 
@@ -275,7 +297,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
     def end_utterance(self) -> typing.Iterable[BaseResult]:
         last_settings = self.settings
 
-        sent_phonemes: PHONEMES_LIST = []
+        sent_phonemes: PHONEMES_LIST_TYPE = []
 
         for result in self._results:
             if isinstance(result, Mimic3Phonemes):
@@ -298,15 +320,44 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
                 yield result
 
         if sent_phonemes:
-            yield self._speak_sentence_phonemes(sent_phonemes)
+            yield self._speak_sentence_phonemes(sent_phonemes, settings=last_settings)
             sent_phonemes.clear()
 
         self._results.clear()
 
-    def preload_voice(self, voice_key: str):
+    # -------------------------------------------------------------------------
-        self._get_or_load_voice(voice_key)
+
+    def _speak_sentence_phonemes(
+        self,
+        sent_phonemes,
+        settings: typing.Optional[Mimic3Settings] = None,
+    ) -> AudioResult:
+        """Synthesize audio from phonemes using given setings"""
+        settings = settings or self.settings
+        voice = self._get_or_load_voice(settings.voice or self.voice)
+        sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes)
+
+        _LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids)
+
+        audio = voice.ids_to_audio(
+            sent_phoneme_ids,
+            speaker=self.speaker,
+            length_scale=settings.length_scale,
+            noise_scale=settings.noise_scale,
+            noise_w=settings.noise_w,
+        )
+
+        audio_bytes = audio.tobytes()
+        return AudioResult(
+            sample_rate_hz=voice.config.audio.sample_rate,
+            audio_bytes=audio_bytes,
+            # 16-bit mono
+            sample_width_bytes=2,
+            num_channels=1,
+        )
 
     def _get_or_load_voice(self, voice_key: str) -> Mimic3Voice:
+        """Get a loaded voice or load from the file system"""
         existing_voice = self._loaded_voices.get(voice_key)
         if existing_voice is not None:
             return existing_voice

mimic3-tts/mimic3_tts/voice.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 import csv
-import itertools
 import logging
 import time
 import typing
@@ -18,6 +17,8 @@ from gruut_ipa import IPA
 from mimic3_tts.config import Phonemizer, TrainingConfig
 from mimic3_tts.utils import audio_float_to_int16
 
+# -----------------------------------------------------------------------------
+
 PHONEME_TYPE = str
 PHONEME_ID_TYPE = int
 WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]]

mimic3-tts/mypy.ini

@@ -5,3 +5,6 @@ ignore_missing_imports = True
 
 [mypy-onnxruntime.*]
 ignore_missing_imports = True
+
+[mypy-xdgenvpy.*]
+ignore_missing_imports = True

mimic3-tts/pylintrc

@@ -33,7 +33,8 @@ disable=
   missing-module-docstring,
   missing-class-docstring,
   missing-function-docstring,
-  import-error
+  import-error,
+  relative-import-beyond-top-level
 
 [FORMAT]
 expected-line-ending-format=LF

mimic3-tts/requirements.txt

@@ -5,3 +5,4 @@ numpy<2.0
 onnxruntime>=1.6,<2.0
 phonemes2ids<2.0
 opentts_abc<1.0
+xdgenvpy>2.0,<3

mimic3-tts/setup.cfg

@@ -13,6 +13,10 @@ ignore =
     D202,
     W504
 
+# F401 import unused
+per-file-ignores =
+    mimic3_tts/__init__.py:F401
+
 [isort]
 multi_line_output = 3
 include_trailing_comma=True

opentts-abc/README.md

@@ -2,3 +2,26 @@
 
 Base classes for open text to speech systems.
 
+
+## SSML
+
+A subset of [SSML](https://www.w3.org/TR/speech-synthesis11/) is supported in `SSMLSpeaker`:
+
+* `<speak>` - wrap around SSML text
+    * `lang` - set language for document
+* `<s>` - sentence (disables automatic sentence breaking)
+    * `lang` - set language for sentence
+* `<w>` / `<token>` - word (disables automatic tokenization)
+* `<voice name="...">` - set voice of inner text
+    * `voice` - name of voice
+* `<say-as interpret-as="">` - force interpretation of inner text
+    * `interpret-as` - way to interpret text (implementation dependent)
+    * `format` - way to format text (implementation dependent)
+* `<break time="">` - Pause for given amount of time
+    * time - seconds ("123s") or milliseconds ("123ms")
+* `<mark name="">` - User-defined mark (written to `--mark-file` or part of `TextToSpeechResult`)
+    * name - name of mark
+* `<sub alias="">` - substitute `alias` for inner text
+* `<phoneme ph="..." alphabet="ipa">` - supply phonemes for inner text
+    * `ph` - phonemes for each word of inner text
+    * `alphabet` - name of phoneme alphabet (usually "ipa")

opentts-abc/opentts_abc/ssml.py

@@ -62,6 +62,9 @@ class SSMLSpeaker:
         self._say_as_format: typing.Optional[str] = None
         self.tts = tts
 
+        self._default_voice = self.tts.voice
+        self._default_lang = self.tts.language
+
     def speak(
         self, ssml: typing.Union[str, etree.Element]
     ) -> typing.Iterable[BaseResult]:
@@ -97,6 +100,8 @@ class SSMLSpeaker:
                     self._handle_end_voice()
                 elif end_tag == "say-as":
                     self._handle_end_say_as()
+                elif end_tag == "lang":
+                    self._handle_end_lang()
                 elif end_tag in {"sub"}:
                     # Handled in handle_text
                     pass
@@ -138,6 +143,8 @@ class SSMLSpeaker:
                     self._handle_begin_voice(elem)
                 elif elem_tag == "say-as":
                     self._handle_begin_say_as(elem)
+                elif elem_tag == "lang":
+                    self._handle_begin_lang(elem)
                 elif elem_tag in {"metadata", "meta"}:
                     self._handle_begin_metadata()
                 else:
@@ -287,6 +294,9 @@ class SSMLSpeaker:
     def _handle_end_speak(self) -> typing.Iterable[BaseResult]:
         """Handle </speak>"""
         LOG.debug("end speak")
+        if self._state == ParsingState.IN_SENTENCE:
+            yield from self._handle_end_sentence()
+
         assert self._state in {ParsingState.DEFAULT}, self._state
 
         yield from self.tts.end_utterance()
@@ -305,10 +315,11 @@ class SSMLSpeaker:
     def _handle_end_voice(self):
         """Handle </voice>"""
         LOG.debug("end voice")
-        voice_name = self._pop_voice()
+        self._pop_voice()
 
         # Restore voice
-        self.tts.voice = voice_name
+        self.tts.voice = self._voice
+        LOG.debug("voice: %s", self._voice)
 
     def _handle_break(self, elem: etree.Element):
         """Handle <break>"""
@@ -348,6 +359,21 @@ class SSMLSpeaker:
         self._say_as_format = None
         self._pop_state()
 
+    def _handle_begin_lang(self, elem: etree.Element):
+        """Handle <lang>"""
+        LOG.debug("begin lang")
+        lang = attrib_no_namespace(elem, "lang")
+
+        LOG.debug("language: %s", lang)
+        self._push_lang(lang)
+
+    def _handle_end_lang(self):
+        """Handle </lang>"""
+        LOG.debug("end lang")
+        self._pop_lang()
+
+        LOG.debug("language: %s", self._lang)
+
     # -------------------------------------------------------------------------
 
     @property
@@ -394,7 +420,7 @@ class SSMLSpeaker:
         if self._lang_stack:
             return self._lang_stack[-1]
 
-        return self.tts.language
+        return self._default_lang
 
     def _push_lang(self, new_lang: str):
         """Push new language on to the stack"""
@@ -405,7 +431,7 @@ class SSMLSpeaker:
         if self._lang_stack:
             return self._lang_stack.pop()
 
-        return self.tts.language
+        return self._default_lang
 
     @property
     def _voice(self) -> typing.Optional[str]:
@@ -413,7 +439,7 @@ class SSMLSpeaker:
         if self._voice_stack:
             return self._voice_stack[-1]
 
-        return self.tts.voice
+        return self._default_voice
 
     def _push_voice(self, new_voice: str):
         """Push new voice on to the stack"""
@@ -424,7 +450,7 @@ class SSMLSpeaker:
         if self._voice_stack:
             return self._voice_stack.pop()
 
-        return self.tts.voice
+        return self._default_voice
 
 
 # -----------------------------------------------------------------------------