jyapayne/mimic3
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add manual minor/major breaks for eSpeak voices

Browse source
This commit is contained in:
Michael Hansen 2022-04-05 10:50:47 -04:00
commit 9fd10f5949
6 changed files with 171 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

26
mimic3-http/mimic3_http/client.py
View file

@ -15,11 +15,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import subprocess
import shlex
import shutil
import logging
import os
import shlex
import shutil
import subprocess
import sys
import tempfile
import typing
@ -118,16 +118,24 @@ def get_args() -> argparse.Namespace:
help="URL of mimic3 HTTP server (default: http://localhost:59125/api/tts)",
)
parser.add_argument(
"--voice", "-v", help="Name of voice (expected in <voices-dir>/<language>)",
"--voice",
"-v",
help="Name of voice (expected in <voices-dir>/<language>)",
)
parser.add_argument(
"--output", "-o", help="Path to write WAV file (default: play audio)",
"--output",
"-o",
help="Path to write WAV file (default: play audio)",
)
parser.add_argument(
"--stdout", action="store_true", help="Write WAV data to stdout",
"--stdout",
action="store_true",
help="Write WAV data to stdout",
)
parser.add_argument(
"--noise-scale", type=float, help="Noise scale [0-1], default is 0.667",
"--noise-scale",
type=float,
help="Noise scale [0-1], default is 0.667",
)
parser.add_argument(
"--length-scale",
@ -135,7 +143,9 @@ def get_args() -> argparse.Namespace:
help="Length scale (1.0 is default speed, 0.5 is 2x faster)",
)
parser.add_argument(
"--noise-w", type=float, help="Variation in cadence [0-1], default is 0.8",
"--noise-w",
type=float,
help="Variation in cadence [0-1], default is 0.8",
)
parser.add_argument(
"--play-program",

3
mimic3-http/mypy.ini
View file

@ -8,3 +8,6 @@ ignore_missing_imports = True
[mypy-setuptools.*]
ignore_missing_imports = True
[mypy-swagger_ui.*]
ignore_missing_imports = True

26
mimic3-tts/mimic3_tts/config.py
View file

@ -29,6 +29,8 @@ from phonemes2ids import BlankBetween
@dataclass
class AudioConfig(DataClassJsonMixin):
"""Audio input/output details"""
filter_length: int = 1024
hop_length: int = 256
win_length: int = 1024
@ -109,6 +111,8 @@ class AudioConfig(DataClassJsonMixin):
@dataclass
class ModelConfig(DataClassJsonMixin):
"""TTS model hyperparameters"""
num_symbols: int = 0
n_speakers: int = 1
@ -141,6 +145,8 @@ class ModelConfig(DataClassJsonMixin):
@dataclass
class PhonemesConfig(DataClassJsonMixin):
"""Phonemes to ids configuration"""
phoneme_separator: str = " "
"""Separator between individual phonemes in CSV input"""
@ -185,21 +191,30 @@ class PhonemesConfig(DataClassJsonMixin):
class Phonemizer(str, Enum):
"""Method used to convert text to phonemes"""
SYMBOLS = "symbols"
GRUUT = "gruut"
ESPEAK = "espeak"
class Aligner(str, Enum):
"""Text/audio aligner"""
KALDI_ALIGN = "kaldi_align"
"""https://github.com/rhasspy/kaldi-align"""
class TextCasing(str, Enum):
"""Casing method applied to text"""
LOWER = "lower"
UPPER = "upper"
class MetadataFormat(str, Enum):
"""Format of training metadata"""
TEXT = "text"
PHONEMES = "phonemes"
PHONEME_IDS = "ids"
@ -207,6 +222,8 @@ class MetadataFormat(str, Enum):
@dataclass
class DatasetConfig:
"""Training dataset configuration"""
name: str
metadata_format: MetadataFormat = MetadataFormat.TEXT
multispeaker: bool = False
@ -228,19 +245,28 @@ class DatasetConfig:
@dataclass
class AlignerConfig:
"""Text/audio alignment configuration"""
aligner: typing.Optional[Aligner] = None
casing: typing.Optional[TextCasing] = None
@dataclass
class InferenceConfig:
"""Inference configuration"""
length_scale: float = 1.0
noise_scale: float = 0.667
noise_w: float = 0.8
minor_break_ms: typing.Optional[int] = None
major_break_ms: typing.Optional[int] = None
@dataclass
class TrainingConfig(DataClassJsonMixin):
"""Master configuration for training"""
seed: int = 1234
epochs: int = 10000
learning_rate: float = 2e-4

20
mimic3-tts/mimic3_tts/tts.py
View file

@ -44,7 +44,7 @@ from .const import (
DEFAULT_VOICES_URL_FORMAT,
)
from .download import VoiceFile, download_voice
from .voice import SPEAKER_TYPE, Mimic3Voice
from .voice import SPEAKER_TYPE, BreakType, Mimic3Voice
_DIR = Path(__file__).parent
@ -293,15 +293,29 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
def speak_text(self, text: str, text_language: typing.Optional[str] = None):
voice = self._get_or_load_voice(self.voice)
for sent_phonemes in voice.text_to_phonemes(text, text_language=text_language):
minor_break_ms = voice.config.inference.major_break_ms
major_break_ms = voice.config.inference.major_break_ms
for sent_phonemes, break_type in voice.text_to_phonemes(
text, text_language=text_language
):
# Utterances have start/end meta phonemes (usually ^ and $)
is_utterance = break_type != BreakType.NONE
self._results.append(
Mimic3Phonemes(
current_settings=deepcopy(self.settings),
phonemes=sent_phonemes,
is_utterance=False,
is_utterance=is_utterance,
)
)
# Add silence if using manual break intervals
if (break_type == BreakType.MAJOR) and (major_break_ms is not None):
self.add_break(major_break_ms)
elif (break_type == BreakType.MINOR) and (minor_break_ms is not None):
self.add_break(minor_break_ms)
# pylint: disable=arguments-differ
def speak_tokens(
self,

53
mimic3-tts/mimic3_tts/voice.py
View file

@ -19,6 +19,7 @@ import platform
import time
import typing
from abc import ABCMeta, abstractmethod
from enum import Enum
from pathlib import Path
from xml.sax.saxutils import escape as xmlescape
@ -34,10 +35,19 @@ from mimic3_tts.utils import audio_float_to_int16
# -----------------------------------------------------------------------------
class BreakType(str, Enum):
NONE = "none"
MINOR = "minor"
MAJOR = "major"
UTTERANCE = "utterance"
PHONEME_TYPE = str
PHONEME_ID_TYPE = int
WORD_PHONEMES_TYPE = typing.List[typing.List[PHONEME_TYPE]]
PHONEME_MAP_TYPE = typing.Dict[PHONEME_TYPE, typing.List[PHONEME_TYPE]]
TEXT_TO_PHONEMES_TYPE = typing.Iterable[typing.Tuple[WORD_PHONEMES_TYPE, BreakType]]
SPEAKER_NAME_TYPE = str
SPEAKER_ID_TYPE = int
@ -48,6 +58,7 @@ DEFAULT_LANGUAGE = "en_US"
_LOGGER = logging.getLogger(__name__)
# -----------------------------------------------------------------------------
@ -71,7 +82,7 @@ class Mimic3Voice(metaclass=ABCMeta):
@abstractmethod
def text_to_phonemes(
self, text: str, text_language: typing.Optional[str] = None
) -> typing.Iterable[WORD_PHONEMES_TYPE]:
) -> TEXT_TO_PHONEMES_TYPE:
"""Convert text into phonemes"""
def word_to_phonemes(
@ -82,7 +93,7 @@ class Mimic3Voice(metaclass=ABCMeta):
) -> typing.List[PHONEME_TYPE]:
"""Convert a single word (with optional role) into phonemes"""
word_phonemes = []
for sent_phonemes in self.text_to_phonemes(
for sent_phonemes, _break_type in self.text_to_phonemes(
word_text, text_language=text_language
):
for sent_word_phonemes in sent_phonemes:
@ -99,7 +110,9 @@ class Mimic3Voice(metaclass=ABCMeta):
) -> WORD_PHONEMES_TYPE:
"""Speak a word or phrase with a specific interpretation/format"""
word_phonemes = []
for sent_phonemes in self.text_to_phonemes(text, text_language=text_language):
for sent_phonemes, _break_type in self.text_to_phonemes(
text, text_language=text_language
):
word_phonemes.extend(sent_phonemes)
return word_phonemes
@ -318,12 +331,12 @@ class GruutVoice(Mimic3Voice):
def text_to_phonemes(
self, text: str, text_language: typing.Optional[str] = None
) -> typing.Iterable[WORD_PHONEMES_TYPE]:
) -> TEXT_TO_PHONEMES_TYPE:
text_language = text_language or self.config.text_language or DEFAULT_LANGUAGE
for sentence in gruut.sentences(text, lang=text_language):
sent_phonemes = [w.phonemes for w in sentence if w.phonemes]
if sent_phonemes:
yield sent_phonemes
yield sent_phonemes, BreakType.NONE
def word_to_phonemes(
self,
@ -389,7 +402,7 @@ class EspeakVoice(Mimic3Voice):
def text_to_phonemes(
self, text: str, text_language: typing.Optional[str] = None
) -> typing.Iterable[WORD_PHONEMES_TYPE]:
) -> TEXT_TO_PHONEMES_TYPE:
phoneme_separator = ""
word_separator = self.config.phonemes.word_separator
@ -406,11 +419,31 @@ class EspeakVoice(Mimic3Voice):
punctuation_separator=phoneme_separator,
)
word_phonemes = [
all_word_phonemes = [
list(IPA.graphemes(wp_str)) for wp_str in phoneme_str.split(word_separator)
]
yield word_phonemes
minor_break = self.config.phonemes.minor_break
major_break = self.config.phonemes.major_break
if minor_break or major_break:
# Split on breaks
sent_phonemes = []
for word_phonemes in all_word_phonemes:
sent_phonemes.append(word_phonemes)
if minor_break and (word_phonemes[-1] == minor_break):
yield sent_phonemes, BreakType.MINOR
sent_phonemes = []
elif major_break and (word_phonemes[-1] == major_break):
yield sent_phonemes, BreakType.MAJOR
sent_phonemes = []
if sent_phonemes:
yield sent_phonemes, BreakType.MAJOR
else:
# No split
yield all_word_phonemes, BreakType.UTTERANCE
def word_to_phonemes(
self,
@ -486,9 +519,9 @@ class SymbolsVoice(Mimic3Voice):
def text_to_phonemes(
self, text: str, text_language: typing.Optional[str] = None
) -> typing.Iterable[WORD_PHONEMES_TYPE]:
) -> TEXT_TO_PHONEMES_TYPE:
word_separator = self.config.phonemes.word_separator
word_phonemes = [
list(IPA.graphemes(wp_str)) for wp_str in text.split(word_separator)
]
yield word_phonemes
yield word_phonemes, BreakType.NONE

64
mimic3-tts/mimic3_tts/voices.json
View file

@ -63,6 +63,36 @@
"speakers": [],
"properties": {}
},
"en_UK/apope_low": {
"files": {
"LICENSE": {
"size_bytes": 46,
"sha256_sum": "2fb2b744e6d96a2e3137d58121670b005718c736d5dde1832c99ff9073f64c00"
},
"README.md": {
"size_bytes": 155,
"sha256_sum": "2e6c39454c35910c6b48518523429ec2e4f27ae8eb763e311877a017db23d4da"
},
"config.json": {
"size_bytes": 3434,
"sha256_sum": "1fdaa1124e02cc177eb776fbc6e08c838b56bd2e86c82d8d7fe434d9337806b0"
},
"generator.onnx": {
"size_bytes": 62792219,
"sha256_sum": "0b5a323500ebd022351db12da2b3aab8cdd47d0826d173e780a58b93604618c9"
},
"phoneme_map.txt": {
"size_bytes": 15,
"sha256_sum": "4003f421fc91ed1d5a343442659db6cf9d58bd1c6d8d771abc1999cc24d7694d"
},
"phonemes.txt": {
"size_bytes": 263,
"sha256_sum": "8f9c3e6ced14d7fc5426e4e1bc7f7cc1037a20a645ca34110abcb76148fa8bfd"
}
},
"speakers": [],
"properties": {}
},
"en_US/cmu-arctic_low": {
"files": {
"LICENSE": {
@ -502,6 +532,40 @@
"speakers": [],
"properties": {}
},
"nl/nathalie_low": {
"files": {
"LICENSE": {
"size_bytes": 7049,
"sha256_sum": "7179683e8000e6bdc9bbc60d85edf0a4ac8e76f951857f54fcb775d5886f1309"
},
"README.md": {
"size_bytes": 180,
"sha256_sum": "15e74dbbdadc8bf7c3a84b1b130f6e5df5f37ffe82880703b27d92e11cff1af8"
},
"SOURCE": {
"size_bytes": 50,
"sha256_sum": "6422e7d891e0d77db7a7cd9643bc8cff4c26f47d652b83b651a0179682ce65c7"
},
"config.json": {
"size_bytes": 3604,
"sha256_sum": "e8e1acf414067657bd377965eb822c51f55483e2ad9a498398e473b0eae4c137"
},
"generator.onnx": {
"size_bytes": 62800663,
"sha256_sum": "1707e33a94538529009c28d0da2a99fe81aa15bd7bb4c2252b3ace0ab704fa71"
},
"phoneme_map.txt": {
"size_bytes": 21,
"sha256_sum": "18b9c5fe46201cd5b0b20e88dc78c64ef3bcaaf87a428386420757f2d0bf7bb3"
},
"phonemes.txt": {
"size_bytes": 336,
"sha256_sum": "355389fee04f97557232cdde7fb8d4cf03ae2aabd7b0b26ed5978ebbf6575dd4"
}
},
"speakers": [],
"properties": {}
},
"nl/rdh_low": {
"files": {
"LICENSE": {