jyapayne/mimic3
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cleanup in mimic3_tts

Browse source
This commit is contained in:
Michael Hansen 2022-03-25 13:20:28 -04:00
commit 245ac200d1
12 changed files with 236 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

5
mimic3-tts/mimic3_tts/__init__.py
View file

@ -1,4 +1,9 @@
from pathlib import Path
from opentts_abc import AudioResult, MarkResult
from opentts_abc.ssml import SSMLSpeaker
from ._resources import __version__
from .tts import Mimic3Settings, Mimic3TextToSpeechSystem
__author__ = "Michael Hansen"

25
mimic3-tts/mimic3_tts/__main__.py
View file

@ -2,16 +2,11 @@
import logging
import wave
logging.basicConfig(level=logging.DEBUG)
from opentts_abc.ssml import SSMLSpeaker
from mimic3_tts.tts import (
AudioResult,
MarkResult,
Mimic3Settings,
Mimic3TextToSpeechSystem,
)
from .tts import AudioResult, MarkResult, Mimic3Settings, Mimic3TextToSpeechSystem
logging.basicConfig(level=logging.DEBUG)
settings = Mimic3Settings()
tts = Mimic3TextToSpeechSystem(settings)
@ -22,21 +17,21 @@ speaker = SSMLSpeaker(tts)
# ssml = '<speak><s><w>Hello</w><w>World</w></s></speak>'
# ssml = '<speak><s>Hello world</s></speak>'
# ssml = '<speak><s><voice name="el_GR/rapunzelina_low"><say-as interpret-as="characters">12</say-as></voice></s></speak>'
ssml = '''
ssml = """
<speak>
<voice name="en_US/amy_low">
<s>
Today is 1/2.
<break time="1s" />
</s>
Today is a test.
This is another test.
</voice>
<voice name="es_ES/carlfm_low">
<s>Soy el <say-as interpret-as="number" format="ordinal">1</say-as>.</s>
<lang xml:lang="es_ES">
Soy el <say-as interpret-as="number" format="ordinal">1</say-as>.
</lang>
</voice>
</speak>
'''
"""
wav_file: wave.Wave_write = wave.open("out.wav", "wb")

19
mimic3-tts/mimic3_tts/_resources.py Normal file
View file

@ -0,0 +1,19 @@
#!/usr/bin/env python3
import os
import typing
from pathlib import Path
try:
import importlib.resources
files = importlib.resources.files
except (ImportError, AttributeError):
# Backport for Python < 3.9
import importlib_resources # type: ignore
files = importlib_resources.files
_PACKAGE = "mimic3_tts"
_DIR = Path(typing.cast(os.PathLike, files(_PACKAGE)))
__version__ = (_DIR / "VERSION").read_text(encoding="utf-8").strip()

12
mimic3-tts/mimic3_tts/config.py
View file

@ -272,18 +272,6 @@ class TrainingConfig(DataClassJsonMixin):
"""Save config as JSON to a file"""
json.dump(self.to_dict(), config_file, indent=4)
def get_speaker_id(self, dataset_name: str, speaker_name: str) -> int:
if self.speaker_id_map is None:
self.speaker_id_map = {}
full_speaker_name = f"{dataset_name}_{speaker_name}"
speaker_id = self.speaker_id_map.get(full_speaker_name)
if speaker_id is None:
speaker_id = len(self.speaker_id_map)
self.speaker_id_map[full_speaker_name] = speaker_id
return speaker_id
@staticmethod
def load(config_file: typing.TextIO) -> "TrainingConfig":
"""Load config from a JSON file"""

219
mimic3-tts/mimic3_tts/tts.py
View file

@ -1,18 +1,13 @@
#!/usr/bin/env python3
"""Implementation of OpenTTS for Mimic 3"""
import itertools
import logging
import time
import typing
from copy import deepcopy
from dataclasses import dataclass, field
from pathlib import Path
from xml.sax.saxutils import escape as xmlescape
import gruut
import numpy as np
import onnxruntime
import phonemes2ids
from gruut.const import LookupPhonemes, WordRole
from gruut_ipa import IPA, Phoneme, guess_phonemes
from gruut_ipa import IPA
from opentts_abc import (
AudioResult,
BaseResult,
@ -24,15 +19,16 @@ from opentts_abc import (
Voice,
Word,
)
from xdgenvpy import XDG
from mimic3_tts.config import TrainingConfig
from mimic3_tts.voice import Mimic3Voice, SPEAKER_TYPE
from .config import TrainingConfig
from .voice import SPEAKER_TYPE, Mimic3Voice
_DIR = Path(__file__).parent
_LOGGER = logging.getLogger(__name__)
PHONEMES_LIST = typing.List[typing.List[str]]
PHONEMES_LIST_TYPE = typing.List[typing.List[str]]
DEFAULT_VOICE = "en_US/vctk_low"
DEFAULT_LANGUAGE = "en_US"
@ -43,25 +39,53 @@ DEFAULT_LANGUAGE = "en_US"
@dataclass
class Mimic3Settings:
"""Settings for Mimic 3 text to speech system"""
voice: typing.Optional[str] = None
"""Default voice key"""
language: typing.Optional[str] = None
"""Default language (e.g., "en_US")"""
voices_directories: typing.Optional[typing.Iterable[typing.Union[str, Path]]] = None
"""Directories to search for voices (<lang>/<voice>)"""
speaker: typing.Optional[SPEAKER_TYPE] = None
"""Default speaker name or id"""
length_scale: typing.Optional[float] = None
"""Default length scale (use voice config if None)"""
noise_scale: typing.Optional[float] = None
"""Default noise scale (use voice config if None)"""
noise_w: typing.Optional[float] = None
"""Default noise W (use voice config if None)"""
text_language: typing.Optional[str] = None
"""Language of text (use voice language if None)"""
sample_rate: int = 22050
"""Sample rate of silence from add_break() in Hertz"""
@dataclass
class Mimic3Phonemes:
"""Pending task to synthesize audio from phonemes with specific settings"""
current_settings: Mimic3Settings
"""Settings used to synthesize audio"""
phonemes: typing.List[typing.List[str]] = field(default_factory=list)
"""Phonemes for synthesis"""
is_utterance: bool = True
"""True if this is the end of a full utterance"""
class VoiceNotFoundError(Exception):
"""Raised if a voice cannot be found"""
def __init__(self, voice: str):
super().__init__(f"Voice not found: {voice}")
@ -78,57 +102,37 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
self._results: typing.List[typing.Union[BaseResult, Mimic3Phonemes]] = []
self._loaded_voices: typing.Dict[str, Mimic3Voice] = {}
@property
def voice(self) -> str:
return self.settings.voice or DEFAULT_VOICE
@voice.setter
def voice(self, new_voice: str):
if new_voice != self.settings.voice:
# Clear speaker on voice change
self.speaker = None
self.settings.voice = new_voice
if "#" in self.settings.voice:
# Split
voice, speaker = self.settings.voice.split("#", maxsplit=1)
self.settings.voice = voice
self.speaker = speaker
@property
def speaker(self) -> typing.Optional[SPEAKER_TYPE]:
return self.settings.speaker
@speaker.setter
def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]):
self.settings.speaker = new_speaker
@property
def language(self) -> str:
return self.settings.language or DEFAULT_LANGUAGE
@language.setter
def language(self, new_language: str):
self.settings.language = new_language
@staticmethod
def get_default_voices_directories() -> typing.List[Path]:
return [_DIR.parent.parent / "voices"]
"""Get list of directories to search for voices by default.
On Linux, this is typically:
- $HOME/.local/share/mimic3
- /usr/local/share/mimic3
- /usr/share/mimic3
"""
data_dirs = [Path(d) / "mimic3" for d in XDG().XDG_DATA_DIRS.split(":")]
return [_DIR.parent.parent / "voices"] + data_dirs
def get_voices(self) -> typing.Iterable[Voice]:
voices_dirs = (
self.settings.voices_directories
or Mimic3TextToSpeechSystem.get_default_voices_directories()
)
"""Returns an iterable of all available voices"""
voices_dirs: typing.Iterable[
typing.Union[str, Path]
] = Mimic3TextToSpeechSystem.get_default_voices_directories()
if self.settings.voices_directories is not None:
voices_dirs = itertools.chain(self.settings.voices_directories, voices_dirs)
# voices/<language>/<voice>/
for voices_dir in voices_dirs:
voices_dir = Path(voices_dir)
if not voices_dir.is_dir():
_LOGGER.debug("Skipping voice directory %s", voices_dir)
continue
_LOGGER.debug("Searching %s for voices", voices_dir)
for lang_dir in voices_dir.iterdir():
if not lang_dir.is_dir():
continue
@ -137,6 +141,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
if not voice_dir.is_dir():
continue
_LOGGER.debug("Voice found in %s", voice_dir)
voice_lang = lang_dir.name
# Load config
@ -176,9 +181,50 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
properties=properties,
)
def preload_voice(self, voice_key: str):
"""Ensure voice is loaded in memory before synthesis"""
self._get_or_load_voice(voice_key)
# -------------------------------------------------------------------------
@property
def voice(self) -> str:
return self.settings.voice or DEFAULT_VOICE
@voice.setter
def voice(self, new_voice: str):
if new_voice != self.settings.voice:
# Clear speaker on voice change
self.speaker = None
self.settings.voice = new_voice
if "#" in self.settings.voice:
# Split
voice, speaker = self.settings.voice.split("#", maxsplit=1)
self.settings.voice = voice
self.speaker = speaker
@property
def speaker(self) -> typing.Optional[SPEAKER_TYPE]:
return self.settings.speaker
@speaker.setter
def speaker(self, new_speaker: typing.Optional[SPEAKER_TYPE]):
self.settings.speaker = new_speaker
@property
def language(self) -> str:
return self.settings.language or DEFAULT_LANGUAGE
@language.setter
def language(self, new_language: str):
self.settings.language = new_language
def begin_utterance(self):
pass
# pylint: disable=arguments-differ
def speak_text(self, text: str, text_language: typing.Optional[str] = None):
voice = self._get_or_load_voice(self.voice)
@ -187,44 +233,18 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
Mimic3Phonemes(
current_settings=deepcopy(self.settings),
phonemes=sent_phonemes,
is_utterance=False,
)
)
def _speak_sentence_phonemes(
self,
sent_phonemes,
settings: typing.Optional[Mimic3Settings] = None,
) -> AudioResult:
settings = settings or self.settings
voice = self._get_or_load_voice(settings.voice or self.voice)
sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes)
_LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids)
audio = voice.ids_to_audio(
sent_phoneme_ids,
speaker=self.speaker,
length_scale=settings.length_scale,
noise_scale=settings.noise_scale,
noise_w=settings.noise_w,
)
audio_bytes = audio.tobytes()
return AudioResult(
sample_rate_hz=voice.config.audio.sample_rate,
audio_bytes=audio_bytes,
# 16-bit mono
sample_width_bytes=2,
num_channels=1,
)
# pylint: disable=arguments-differ
def speak_tokens(
self,
tokens: typing.Iterable[BaseToken],
text_language: typing.Optional[str] = None,
):
voice = self._get_or_load_voice(self.voice)
token_phonemes: PHONEMES_LIST = []
token_phonemes: PHONEMES_LIST_TYPE = []
for token in tokens:
if isinstance(token, Word):
@ -250,7 +270,9 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
if token_phonemes:
self._results.append(
Mimic3Phonemes(
current_settings=deepcopy(self.settings), phonemes=token_phonemes
current_settings=deepcopy(self.settings),
phonemes=token_phonemes,
is_utterance=False,
)
)
@ -275,7 +297,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
def end_utterance(self) -> typing.Iterable[BaseResult]:
last_settings = self.settings
sent_phonemes: PHONEMES_LIST = []
sent_phonemes: PHONEMES_LIST_TYPE = []
for result in self._results:
if isinstance(result, Mimic3Phonemes):
@ -298,15 +320,44 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
yield result
if sent_phonemes:
yield self._speak_sentence_phonemes(sent_phonemes)
yield self._speak_sentence_phonemes(sent_phonemes, settings=last_settings)
sent_phonemes.clear()
self._results.clear()
def preload_voice(self, voice_key: str):
self._get_or_load_voice(voice_key)
# -------------------------------------------------------------------------
def _speak_sentence_phonemes(
self,
sent_phonemes,
settings: typing.Optional[Mimic3Settings] = None,
) -> AudioResult:
"""Synthesize audio from phonemes using given setings"""
settings = settings or self.settings
voice = self._get_or_load_voice(settings.voice or self.voice)
sent_phoneme_ids = voice.phonemes_to_ids(sent_phonemes)
_LOGGER.debug("phonemes=%s, ids=%s", sent_phonemes, sent_phoneme_ids)
audio = voice.ids_to_audio(
sent_phoneme_ids,
speaker=self.speaker,
length_scale=settings.length_scale,
noise_scale=settings.noise_scale,
noise_w=settings.noise_w,
)
audio_bytes = audio.tobytes()
return AudioResult(
sample_rate_hz=voice.config.audio.sample_rate,
audio_bytes=audio_bytes,
# 16-bit mono
sample_width_bytes=2,
num_channels=1,
)
def _get_or_load_voice(self, voice_key: str) -> Mimic3Voice:
"""Get a loaded voice or load from the file system"""
existing_voice = self._loaded_voices.get(voice_key)
if existing_voice is not None:
return existing_voice

3
mimic3-tts/mimic3_tts/voice.py
View file

@ -1,6 +1,5 @@
#!/usr/bin/env python3
import csv
import itertools
import logging
import time
import typing
@ -18,6 +17,8 @@ from gruut_ipa import IPA
from mimic3_tts.config import Phonemizer, TrainingConfig
from mimic3_tts.utils import audio_float_to_int16
# -----------------------------------------------------------------------------
PHONEME_TYPE = str
PHONEME_ID_TYPE = int
WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]]

3
mimic3-tts/mypy.ini
View file

@ -5,3 +5,6 @@ ignore_missing_imports = True
[mypy-onnxruntime.*]
ignore_missing_imports = True
[mypy-xdgenvpy.*]
ignore_missing_imports = True

3
mimic3-tts/pylintrc
View file

@ -33,7 +33,8 @@ disable=
missing-module-docstring,
missing-class-docstring,
missing-function-docstring,
import-error
import-error,
relative-import-beyond-top-level
[FORMAT]
expected-line-ending-format=LF

1
mimic3-tts/requirements.txt
View file

@ -5,3 +5,4 @@ numpy<2.0
onnxruntime>=1.6,<2.0
phonemes2ids<2.0
opentts_abc<1.0
xdgenvpy>2.0,<3

4
mimic3-tts/setup.cfg
View file

@ -13,6 +13,10 @@ ignore =
D202,
W504
# F401 import unused
per-file-ignores =
mimic3_tts/__init__.py:F401
[isort]
multi_line_output = 3
include_trailing_comma=True

23
opentts-abc/README.md
View file

@ -2,3 +2,26 @@
Base classes for open text to speech systems.
## SSML
A subset of [SSML](https://www.w3.org/TR/speech-synthesis11/) is supported in `SSMLSpeaker`:
* `<speak>` - wrap around SSML text
* `lang` - set language for document
* `<s>` - sentence (disables automatic sentence breaking)
* `lang` - set language for sentence
* `<w>` / `<token>` - word (disables automatic tokenization)
* `<voice name="...">` - set voice of inner text
* `voice` - name of voice
* `<say-as interpret-as="">` - force interpretation of inner text
* `interpret-as` - way to interpret text (implementation dependent)
* `format` - way to format text (implementation dependent)
* `<break time="">` - Pause for given amount of time
* time - seconds ("123s") or milliseconds ("123ms")
* `<mark name="">` - User-defined mark (written to `--mark-file` or part of `TextToSpeechResult`)
* name - name of mark
* `<sub alias="">` - substitute `alias` for inner text
* `<phoneme ph="..." alphabet="ipa">` - supply phonemes for inner text
* `ph` - phonemes for each word of inner text
* `alphabet` - name of phoneme alphabet (usually "ipa")

38
opentts-abc/opentts_abc/ssml.py
View file

@ -62,6 +62,9 @@ class SSMLSpeaker:
self._say_as_format: typing.Optional[str] = None
self.tts = tts
self._default_voice = self.tts.voice
self._default_lang = self.tts.language
def speak(
self, ssml: typing.Union[str, etree.Element]
) -> typing.Iterable[BaseResult]:
@ -97,6 +100,8 @@ class SSMLSpeaker:
self._handle_end_voice()
elif end_tag == "say-as":
self._handle_end_say_as()
elif end_tag == "lang":
self._handle_end_lang()
elif end_tag in {"sub"}:
# Handled in handle_text
pass
@ -138,6 +143,8 @@ class SSMLSpeaker:
self._handle_begin_voice(elem)
elif elem_tag == "say-as":
self._handle_begin_say_as(elem)
elif elem_tag == "lang":
self._handle_begin_lang(elem)
elif elem_tag in {"metadata", "meta"}:
self._handle_begin_metadata()
else:
@ -287,6 +294,9 @@ class SSMLSpeaker:
def _handle_end_speak(self) -> typing.Iterable[BaseResult]:
"""Handle </speak>"""
LOG.debug("end speak")
if self._state == ParsingState.IN_SENTENCE:
yield from self._handle_end_sentence()
assert self._state in {ParsingState.DEFAULT}, self._state
yield from self.tts.end_utterance()
@ -305,10 +315,11 @@ class SSMLSpeaker:
def _handle_end_voice(self):
"""Handle </voice>"""
LOG.debug("end voice")
voice_name = self._pop_voice()
self._pop_voice()
# Restore voice
self.tts.voice = voice_name
self.tts.voice = self._voice
LOG.debug("voice: %s", self._voice)
def _handle_break(self, elem: etree.Element):
"""Handle <break>"""
@ -348,6 +359,21 @@ class SSMLSpeaker:
self._say_as_format = None
self._pop_state()
def _handle_begin_lang(self, elem: etree.Element):
"""Handle <lang>"""
LOG.debug("begin lang")
lang = attrib_no_namespace(elem, "lang")
LOG.debug("language: %s", lang)
self._push_lang(lang)
def _handle_end_lang(self):
"""Handle </lang>"""
LOG.debug("end lang")
self._pop_lang()
LOG.debug("language: %s", self._lang)
# -------------------------------------------------------------------------
@property
@ -394,7 +420,7 @@ class SSMLSpeaker:
if self._lang_stack:
return self._lang_stack[-1]
return self.tts.language
return self._default_lang
def _push_lang(self, new_lang: str):
"""Push new language on to the stack"""
@ -405,7 +431,7 @@ class SSMLSpeaker:
if self._lang_stack:
return self._lang_stack.pop()
return self.tts.language
return self._default_lang
@property
def _voice(self) -> typing.Optional[str]:
@ -413,7 +439,7 @@ class SSMLSpeaker:
if self._voice_stack:
return self._voice_stack[-1]
return self.tts.voice
return self._default_voice
def _push_voice(self, new_voice: str):
"""Push new voice on to the stack"""
@ -424,7 +450,7 @@ class SSMLSpeaker:
if self._voice_stack:
return self._voice_stack.pop()
return self.tts.voice
return self._default_voice
# -----------------------------------------------------------------------------