diff --git a/mimic3-tts/mimic3_tts/__init__.py b/mimic3-tts/mimic3_tts/__init__.py index a0b8eac..460732e 100644 --- a/mimic3-tts/mimic3_tts/__init__.py +++ b/mimic3-tts/mimic3_tts/__init__.py @@ -1,4 +1,9 @@ +from pathlib import Path + from opentts_abc import AudioResult, MarkResult from opentts_abc.ssml import SSMLSpeaker +from ._resources import __version__ from .tts import Mimic3Settings, Mimic3TextToSpeechSystem + +__author__ = "Michael Hansen" diff --git a/mimic3-tts/mimic3_tts/__main__.py b/mimic3-tts/mimic3_tts/__main__.py index f145da0..edf45ec 100644 --- a/mimic3-tts/mimic3_tts/__main__.py +++ b/mimic3-tts/mimic3_tts/__main__.py @@ -2,16 +2,11 @@ import logging import wave -logging.basicConfig(level=logging.DEBUG) - from opentts_abc.ssml import SSMLSpeaker -from mimic3_tts.tts import ( - AudioResult, - MarkResult, - Mimic3Settings, - Mimic3TextToSpeechSystem, -) +from .tts import AudioResult, MarkResult, Mimic3Settings, Mimic3TextToSpeechSystem + +logging.basicConfig(level=logging.DEBUG) settings = Mimic3Settings() tts = Mimic3TextToSpeechSystem(settings) @@ -22,21 +17,21 @@ speaker = SSMLSpeaker(tts) # ssml = 'HelloWorld' # ssml = 'Hello world' # ssml = '12' -ssml = ''' +ssml = """ - - Today is 1/2. - - + Today is a test. + This is another test. - Soy el 1. + + Soy el 1. + -''' +""" wav_file: wave.Wave_write = wave.open("out.wav", "wb") diff --git a/mimic3-tts/mimic3_tts/_resources.py b/mimic3-tts/mimic3_tts/_resources.py new file mode 100644 index 0000000..ff0b076 --- /dev/null +++ b/mimic3-tts/mimic3_tts/_resources.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import os +import typing +from pathlib import Path + +try: + import importlib.resources + + files = importlib.resources.files +except (ImportError, AttributeError): + # Backport for Python < 3.9 + import importlib_resources # type: ignore + + files = importlib_resources.files + +_PACKAGE = "mimic3_tts" +_DIR = Path(typing.cast(os.PathLike, files(_PACKAGE))) + +__version__ = (_DIR / "VERSION").read_text(encoding="utf-8").strip() diff --git a/mimic3-tts/mimic3_tts/config.py b/mimic3-tts/mimic3_tts/config.py index 87edf5a..7def58a 100644 --- a/mimic3-tts/mimic3_tts/config.py +++ b/mimic3-tts/mimic3_tts/config.py @@ -272,18 +272,6 @@ class TrainingConfig(DataClassJsonMixin): """Save config as JSON to a file""" json.dump(self.to_dict(), config_file, indent=4) - def get_speaker_id(self, dataset_name: str, speaker_name: str) -> int: - if self.speaker_id_map is None: - self.speaker_id_map = {} - - full_speaker_name = f"{dataset_name}_{speaker_name}" - speaker_id = self.speaker_id_map.get(full_speaker_name) - if speaker_id is None: - speaker_id = len(self.speaker_id_map) - self.speaker_id_map[full_speaker_name] = speaker_id - - return speaker_id - @staticmethod def load(config_file: typing.TextIO) -> "TrainingConfig": """Load config from a JSON file""" diff --git a/mimic3-tts/mimic3_tts/tts.py b/mimic3-tts/mimic3_tts/tts.py index 585af9b..b3bc827 100644 --- a/mimic3-tts/mimic3_tts/tts.py +++ b/mimic3-tts/mimic3_tts/tts.py @@ -1,18 +1,13 @@ #!/usr/bin/env python3 +"""Implementation of OpenTTS for Mimic 3""" +import itertools import logging -import time import typing from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path -from xml.sax.saxutils import escape as xmlescape -import gruut -import numpy as np -import onnxruntime -import phonemes2ids -from gruut.const import LookupPhonemes, WordRole -from gruut_ipa import IPA, Phoneme, guess_phonemes +from gruut_ipa import IPA from opentts_abc import ( AudioResult, BaseResult, @@ -24,15 +19,16 @@ from opentts_abc import ( Voice, Word, ) +from xdgenvpy import XDG -from mimic3_tts.config import TrainingConfig -from mimic3_tts.voice import Mimic3Voice, SPEAKER_TYPE +from .config import TrainingConfig +from .voice import SPEAKER_TYPE, Mimic3Voice _DIR = Path(__file__).parent _LOGGER = logging.getLogger(__name__) -PHONEMES_LIST = typing.List[typing.List[str]] +PHONEMES_LIST_TYPE = typing.List[typing.List[str]] DEFAULT_VOICE = "en_US/vctk_low" DEFAULT_LANGUAGE = "en_US" @@ -43,25 +39,53 @@ DEFAULT_LANGUAGE = "en_US" @dataclass class Mimic3Settings: + """Settings for Mimic 3 text to speech system""" + voice: typing.Optional[str] = None + """Default voice key""" + language: typing.Optional[str] = None + """Default language (e.g., "en_US")""" + voices_directories: typing.Optional[typing.Iterable[typing.Union[str, Path]]] = None + """Directories to search for voices (/)""" + speaker: typing.Optional[SPEAKER_TYPE] = None + """Default speaker name or id""" + length_scale: typing.Optional[float] = None + """Default length scale (use voice config if None)""" + noise_scale: typing.Optional[float] = None + """Default noise scale (use voice config if None)""" + noise_w: typing.Optional[float] = None + """Default noise W (use voice config if None)""" + text_language: typing.Optional[str] = None + """Language of text (use voice language if None)""" + sample_rate: int = 22050 + """Sample rate of silence from add_break() in Hertz""" @dataclass class Mimic3Phonemes: + """Pending task to synthesize audio from phonemes with specific settings""" + current_settings: Mimic3Settings + """Settings used to synthesize audio""" + phonemes: typing.List[typing.List[str]] = field(default_factory=list) + """Phonemes for synthesis""" + is_utterance: bool = True + """True if this is the end of a full utterance""" class VoiceNotFoundError(Exception): + """Raised if a voice cannot be found""" + def __init__(self, voice: str): super().__init__(f"Voice not found: {voice}") @@ -78,57 +102,37 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): self._results: typing.List[typing.Union[BaseResult, Mimic3Phonemes]] = [] self._loaded_voices: typing.Dict[str, Mimic3Voice] = {} - @property - def voice(self) -> str: - return self.settings.voice or DEFAULT_VOICE - - @voice.setter - def voice(self, new_voice: str): - if new_voice != self.settings.voice: - # Clear speaker on voice change - self.speaker = None - - self.settings.voice = new_voice - - if "#" in self.settings.voice: - # Split - voice, speaker = self.settings.voice.split("#", maxsplit=1) - self.settings.voice = voice - self.speaker = speaker - - @property - def speaker(self) -> typing.Optional[SPEAKER_TYPE]: - return self.settings.speaker - - @speaker.setter - def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]): - self.settings.speaker = new_speaker - - @property - def language(self) -> str: - return self.settings.language or DEFAULT_LANGUAGE - - @language.setter - def language(self, new_language: str): - self.settings.language = new_language - @staticmethod def get_default_voices_directories() -> typing.List[Path]: - return [_DIR.parent.parent / "voices"] + """Get list of directories to search for voices by default. + + On Linux, this is typically: + - $HOME/.local/share/mimic3 + - /usr/local/share/mimic3 + - /usr/share/mimic3 + """ + data_dirs = [Path(d) / "mimic3" for d in XDG().XDG_DATA_DIRS.split(":")] + return [_DIR.parent.parent / "voices"] + data_dirs def get_voices(self) -> typing.Iterable[Voice]: - voices_dirs = ( - self.settings.voices_directories - or Mimic3TextToSpeechSystem.get_default_voices_directories() - ) + """Returns an iterable of all available voices""" + voices_dirs: typing.Iterable[ + typing.Union[str, Path] + ] = Mimic3TextToSpeechSystem.get_default_voices_directories() + + if self.settings.voices_directories is not None: + voices_dirs = itertools.chain(self.settings.voices_directories, voices_dirs) # voices/// for voices_dir in voices_dirs: voices_dir = Path(voices_dir) if not voices_dir.is_dir(): + _LOGGER.debug("Skipping voice directory %s", voices_dir) continue + _LOGGER.debug("Searching %s for voices", voices_dir) + for lang_dir in voices_dir.iterdir(): if not lang_dir.is_dir(): continue @@ -137,6 +141,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): if not voice_dir.is_dir(): continue + _LOGGER.debug("Voice found in %s", voice_dir) voice_lang = lang_dir.name # Load config @@ -176,9 +181,50 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): properties=properties, ) + def preload_voice(self, voice_key: str): + """Ensure voice is loaded in memory before synthesis""" + self._get_or_load_voice(voice_key) + + # ------------------------------------------------------------------------- + + @property + def voice(self) -> str: + return self.settings.voice or DEFAULT_VOICE + + @voice.setter + def voice(self, new_voice: str): + if new_voice != self.settings.voice: + # Clear speaker on voice change + self.speaker = None + + self.settings.voice = new_voice + + if "#" in self.settings.voice: + # Split + voice, speaker = self.settings.voice.split("#", maxsplit=1) + self.settings.voice = voice + self.speaker = speaker + + @property + def speaker(self) -> typing.Optional[SPEAKER_TYPE]: + return self.settings.speaker + + @speaker.setter + def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]): + self.settings.speaker = new_speaker + + @property + def language(self) -> str: + return self.settings.language or DEFAULT_LANGUAGE + + @language.setter + def language(self, new_language: str): + self.settings.language = new_language + def begin_utterance(self): pass + # pylint: disable=arguments-differ def speak_text(self, text: str, text_language: typing.Optional[str] = None): voice = self._get_or_load_voice(self.voice) @@ -187,44 +233,18 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): Mimic3Phonemes( current_settings=deepcopy(self.settings), phonemes=sent_phonemes, + is_utterance=False, ) ) - def _speak_sentence_phonemes( - self, - sent_phonemes, - settings: typing.Optional[Mimic3Settings] = None, - ) -> AudioResult: - settings = settings or self.settings - voice = self._get_or_load_voice(settings.voice or self.voice) - sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes) - - _LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids) - - audio = voice.ids_to_audio( - sent_phoneme_ids, - speaker=self.speaker, - length_scale=settings.length_scale, - noise_scale=settings.noise_scale, - noise_w=settings.noise_w, - ) - - audio_bytes = audio.tobytes() - return AudioResult( - sample_rate_hz=voice.config.audio.sample_rate, - audio_bytes=audio_bytes, - # 16-bit mono - sample_width_bytes=2, - num_channels=1, - ) - + # pylint: disable=arguments-differ def speak_tokens( self, tokens: typing.Iterable[BaseToken], text_language: typing.Optional[str] = None, ): voice = self._get_or_load_voice(self.voice) - token_phonemes: PHONEMES_LIST = [] + token_phonemes: PHONEMES_LIST_TYPE = [] for token in tokens: if isinstance(token, Word): @@ -250,7 +270,9 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): if token_phonemes: self._results.append( Mimic3Phonemes( - current_settings=deepcopy(self.settings), phonemes=token_phonemes + current_settings=deepcopy(self.settings), + phonemes=token_phonemes, + is_utterance=False, ) ) @@ -275,7 +297,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): def end_utterance(self) -> typing.Iterable[BaseResult]: last_settings = self.settings - sent_phonemes: PHONEMES_LIST = [] + sent_phonemes: PHONEMES_LIST_TYPE = [] for result in self._results: if isinstance(result, Mimic3Phonemes): @@ -298,15 +320,44 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem): yield result if sent_phonemes: - yield self._speak_sentence_phonemes(sent_phonemes) + yield self._speak_sentence_phonemes(sent_phonemes, settings=last_settings) sent_phonemes.clear() self._results.clear() - def preload_voice(self, voice_key: str): - self._get_or_load_voice(voice_key) + # ------------------------------------------------------------------------- + + def _speak_sentence_phonemes( + self, + sent_phonemes, + settings: typing.Optional[Mimic3Settings] = None, + ) -> AudioResult: + """Synthesize audio from phonemes using given setings""" + settings = settings or self.settings + voice = self._get_or_load_voice(settings.voice or self.voice) + sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes) + + _LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids) + + audio = voice.ids_to_audio( + sent_phoneme_ids, + speaker=self.speaker, + length_scale=settings.length_scale, + noise_scale=settings.noise_scale, + noise_w=settings.noise_w, + ) + + audio_bytes = audio.tobytes() + return AudioResult( + sample_rate_hz=voice.config.audio.sample_rate, + audio_bytes=audio_bytes, + # 16-bit mono + sample_width_bytes=2, + num_channels=1, + ) def _get_or_load_voice(self, voice_key: str) -> Mimic3Voice: + """Get a loaded voice or load from the file system""" existing_voice = self._loaded_voices.get(voice_key) if existing_voice is not None: return existing_voice diff --git a/mimic3-tts/mimic3_tts/voice.py b/mimic3-tts/mimic3_tts/voice.py index e22adfb..82704f4 100644 --- a/mimic3-tts/mimic3_tts/voice.py +++ b/mimic3-tts/mimic3_tts/voice.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import csv -import itertools import logging import time import typing @@ -18,6 +17,8 @@ from gruut_ipa import IPA from mimic3_tts.config import Phonemizer, TrainingConfig from mimic3_tts.utils import audio_float_to_int16 +# ----------------------------------------------------------------------------- + PHONEME_TYPE = str PHONEME_ID_TYPE = int WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]] diff --git a/mimic3-tts/mypy.ini b/mimic3-tts/mypy.ini index 916fdd7..99a4d79 100644 --- a/mimic3-tts/mypy.ini +++ b/mimic3-tts/mypy.ini @@ -5,3 +5,6 @@ ignore_missing_imports = True [mypy-onnxruntime.*] ignore_missing_imports = True + +[mypy-xdgenvpy.*] +ignore_missing_imports = True diff --git a/mimic3-tts/pylintrc b/mimic3-tts/pylintrc index 9a76583..792fbe6 100644 --- a/mimic3-tts/pylintrc +++ b/mimic3-tts/pylintrc @@ -33,7 +33,8 @@ disable= missing-module-docstring, missing-class-docstring, missing-function-docstring, - import-error + import-error, + relative-import-beyond-top-level [FORMAT] expected-line-ending-format=LF diff --git a/mimic3-tts/requirements.txt b/mimic3-tts/requirements.txt index ad09284..4cc949b 100644 --- a/mimic3-tts/requirements.txt +++ b/mimic3-tts/requirements.txt @@ -5,3 +5,4 @@ numpy<2.0 onnxruntime>=1.6,<2.0 phonemes2ids<2.0 opentts_abc<1.0 +xdgenvpy>2.0,<3 diff --git a/mimic3-tts/setup.cfg b/mimic3-tts/setup.cfg index 0076bbf..d09a842 100644 --- a/mimic3-tts/setup.cfg +++ b/mimic3-tts/setup.cfg @@ -13,6 +13,10 @@ ignore = D202, W504 +# F401 import unused +per-file-ignores = + mimic3_tts/__init__.py:F401 + [isort] multi_line_output = 3 include_trailing_comma=True diff --git a/opentts-abc/README.md b/opentts-abc/README.md index 76476b9..d15fcd9 100644 --- a/opentts-abc/README.md +++ b/opentts-abc/README.md @@ -2,3 +2,26 @@ Base classes for open text to speech systems. + +## SSML + +A subset of [SSML](https://www.w3.org/TR/speech-synthesis11/) is supported in `SSMLSpeaker`: + +* `` - wrap around SSML text + * `lang` - set language for document +* `` - sentence (disables automatic sentence breaking) + * `lang` - set language for sentence +* `` / `` - word (disables automatic tokenization) +* `` - set voice of inner text + * `voice` - name of voice +* `` - force interpretation of inner text + * `interpret-as` - way to interpret text (implementation dependent) + * `format` - way to format text (implementation dependent) +* `` - Pause for given amount of time + * time - seconds ("123s") or milliseconds ("123ms") +* `` - User-defined mark (written to `--mark-file` or part of `TextToSpeechResult`) + * name - name of mark +* `` - substitute `alias` for inner text +* `` - supply phonemes for inner text + * `ph` - phonemes for each word of inner text + * `alphabet` - name of phoneme alphabet (usually "ipa") diff --git a/opentts-abc/opentts_abc/ssml.py b/opentts-abc/opentts_abc/ssml.py index 0921f46..5cf177c 100644 --- a/opentts-abc/opentts_abc/ssml.py +++ b/opentts-abc/opentts_abc/ssml.py @@ -62,6 +62,9 @@ class SSMLSpeaker: self._say_as_format: typing.Optional[str] = None self.tts = tts + self._default_voice = self.tts.voice + self._default_lang = self.tts.language + def speak( self, ssml: typing.Union[str, etree.Element] ) -> typing.Iterable[BaseResult]: @@ -97,6 +100,8 @@ class SSMLSpeaker: self._handle_end_voice() elif end_tag == "say-as": self._handle_end_say_as() + elif end_tag == "lang": + self._handle_end_lang() elif end_tag in {"sub"}: # Handled in handle_text pass @@ -138,6 +143,8 @@ class SSMLSpeaker: self._handle_begin_voice(elem) elif elem_tag == "say-as": self._handle_begin_say_as(elem) + elif elem_tag == "lang": + self._handle_begin_lang(elem) elif elem_tag in {"metadata", "meta"}: self._handle_begin_metadata() else: @@ -287,6 +294,9 @@ class SSMLSpeaker: def _handle_end_speak(self) -> typing.Iterable[BaseResult]: """Handle """ LOG.debug("end speak") + if self._state == ParsingState.IN_SENTENCE: + yield from self._handle_end_sentence() + assert self._state in {ParsingState.DEFAULT}, self._state yield from self.tts.end_utterance() @@ -305,10 +315,11 @@ class SSMLSpeaker: def _handle_end_voice(self): """Handle """ LOG.debug("end voice") - voice_name = self._pop_voice() + self._pop_voice() # Restore voice - self.tts.voice = voice_name + self.tts.voice = self._voice + LOG.debug("voice: %s", self._voice) def _handle_break(self, elem: etree.Element): """Handle """ @@ -348,6 +359,21 @@ class SSMLSpeaker: self._say_as_format = None self._pop_state() + def _handle_begin_lang(self, elem: etree.Element): + """Handle """ + LOG.debug("begin lang") + lang = attrib_no_namespace(elem, "lang") + + LOG.debug("language: %s", lang) + self._push_lang(lang) + + def _handle_end_lang(self): + """Handle """ + LOG.debug("end lang") + self._pop_lang() + + LOG.debug("language: %s", self._lang) + # ------------------------------------------------------------------------- @property @@ -394,7 +420,7 @@ class SSMLSpeaker: if self._lang_stack: return self._lang_stack[-1] - return self.tts.language + return self._default_lang def _push_lang(self, new_lang: str): """Push new language on to the stack""" @@ -405,7 +431,7 @@ class SSMLSpeaker: if self._lang_stack: return self._lang_stack.pop() - return self.tts.language + return self._default_lang @property def _voice(self) -> typing.Optional[str]: @@ -413,7 +439,7 @@ class SSMLSpeaker: if self._voice_stack: return self._voice_stack[-1] - return self.tts.voice + return self._default_voice def _push_voice(self, new_voice: str): """Push new voice on to the stack""" @@ -424,7 +450,7 @@ class SSMLSpeaker: if self._voice_stack: return self._voice_stack.pop() - return self.tts.voice + return self._default_voice # -----------------------------------------------------------------------------