Fixing marks in CLI
This commit is contained in:
Michael Hansen
parent
355a467c5e
commit
50bc04b987
7 changed files with 237 additions and 665 deletions
|
|
@ -22,7 +22,7 @@ from pathlib import Path
|
|||
from queue import Queue
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from mimic3_tts import Mimic3TextToSpeechSystem
|
||||
from mimic3_tts import Mimic3TextToSpeechSystem, BaseResult
|
||||
|
||||
|
||||
_DIR = Path(__file__).parent
|
||||
|
|
@ -46,8 +46,15 @@ class CommandLineInterfaceState:
|
|||
sample_width_bytes: int = 2
|
||||
num_channels: int = 1
|
||||
|
||||
raw_queue: typing.Optional["Queue[typing.Optional[bytes]]"] = None
|
||||
raw_stream_thread: typing.Optional[threading.Thread] = None
|
||||
result_queue: typing.Optional["Queue[BaseResult]"] = None
|
||||
result_thread: typing.Optional[threading.Thread] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResultToProcess:
|
||||
result: "BaseResult"
|
||||
line: str
|
||||
line_id: str = ""
|
||||
|
||||
|
||||
class OutputNaming(str, Enum):
|
||||
|
|
@ -133,8 +140,10 @@ def initialize_args(state: CommandLineInterfaceState):
|
|||
state.mark_writer = open( # pylint: disable=consider-using-with
|
||||
args.mark_file, "w", encoding="utf-8"
|
||||
)
|
||||
else:
|
||||
elif args.stdout:
|
||||
state.mark_writer = sys.stderr
|
||||
else:
|
||||
state.mark_writer = sys.stdout
|
||||
|
||||
if args.seed is not None:
|
||||
_LOGGER.debug("Setting random seed to %s", args.seed)
|
||||
|
|
@ -201,181 +210,138 @@ def initialize_tts(state: CommandLineInterfaceState):
|
|||
|
||||
args = state.args
|
||||
|
||||
# TODO: voice/speaker
|
||||
state.tts = Mimic3TextToSpeechSystem(Mimic3Settings())
|
||||
|
||||
if state.args.voice:
|
||||
# Set default voice
|
||||
state.tts.voice = state.args.voice
|
||||
|
||||
# max_thread_workers: typing.Optional[int] = None
|
||||
if state.args.preload_voice:
|
||||
for voice_key in state.args.preload_voice:
|
||||
_LOGGER.debug("Preloading voice: %s", voice_key)
|
||||
state.tts.preload_voice(voice_key)
|
||||
|
||||
# if args.max_thread_workers is not None:
|
||||
# max_thread_workers = (
|
||||
# None if args.max_thread_workers < 1 else args.max_thread_workers
|
||||
# )
|
||||
# elif args.raw_stream:
|
||||
# # Faster time to first audio
|
||||
# max_thread_workers = 2
|
||||
state.result_queue = Queue(maxsize=args.result_queue_size)
|
||||
|
||||
# executor = ThreadPoolExecutor(max_workers=max_thread_workers)
|
||||
|
||||
# if os.isatty(sys.stdout.fileno()):
|
||||
# if (not args.output_dir) and (not args.raw_stream):
|
||||
# # No where else for the audio to go
|
||||
# args.interactive = True
|
||||
|
||||
if args.raw_stream:
|
||||
# Output in a separate thread to avoid blocking audio processing
|
||||
state.raw_queue = Queue(maxsize=args.raw_stream_queue_size)
|
||||
|
||||
def output_raw_stream():
|
||||
while True:
|
||||
audio = state.raw_queue.get()
|
||||
if audio is None:
|
||||
break
|
||||
|
||||
_LOGGER.debug(
|
||||
"Writing %s byte(s) of 16-bit 22050Hz mono PCM to stdout",
|
||||
len(audio),
|
||||
)
|
||||
sys.stdout.buffer.write(audio)
|
||||
sys.stdout.buffer.flush()
|
||||
|
||||
state.raw_stream_thread = threading.Thread(
|
||||
target=output_raw_stream, daemon=True
|
||||
)
|
||||
state.raw_stream_thread.start()
|
||||
state.result_thread = threading.Thread(
|
||||
target=process_result, daemon=True, args=(state,)
|
||||
)
|
||||
state.result_thread.start()
|
||||
|
||||
|
||||
def process_line(line_id: str, line: str, state: CommandLineInterfaceState):
|
||||
from mimic3_tts import AudioResult, MarkResult
|
||||
def process_result(state: CommandLineInterfaceState):
|
||||
try:
|
||||
from mimic3_tts import AudioResult, MarkResult
|
||||
|
||||
assert state.result_queue is not None
|
||||
args = state.args
|
||||
|
||||
while True:
|
||||
result_todo = state.result_queue.get()
|
||||
if result_todo is None:
|
||||
break
|
||||
|
||||
try:
|
||||
result = result_todo.result
|
||||
line = result_todo.line
|
||||
line_id = result_todo.line_id
|
||||
|
||||
if isinstance(result, AudioResult):
|
||||
if args.interactive or args.output_dir:
|
||||
# Convert to WAV audio
|
||||
wav_bytes: typing.Optional[bytes] = None
|
||||
if args.interactive:
|
||||
if args.stdout:
|
||||
# Write audio to stdout
|
||||
sys.stdout.buffer.write(result.audio_bytes)
|
||||
sys.stdout.buffer.flush()
|
||||
else:
|
||||
# Play sound
|
||||
if not wav_bytes:
|
||||
wav_bytes = result.to_wav_bytes()
|
||||
|
||||
if wav_bytes:
|
||||
play_wav_bytes(wav_bytes)
|
||||
|
||||
if args.output_dir:
|
||||
if not wav_bytes:
|
||||
wav_bytes = result.to_wav_bytes()
|
||||
|
||||
# Determine file name
|
||||
if args.output_naming == OutputNaming.TEXT:
|
||||
# Use text itself
|
||||
file_name = line.strip().replace(" ", "_")
|
||||
file_name = file_name.translate(
|
||||
str.maketrans(
|
||||
"", "", string.punctuation.replace("_", "")
|
||||
)
|
||||
)
|
||||
elif args.output_naming == OutputNaming.TIME:
|
||||
# Use timestamp
|
||||
file_name = str(time.time())
|
||||
elif args.output_naming == OutputNaming.ID:
|
||||
file_name = line_id
|
||||
|
||||
assert file_name, f"No file name for text: {line}"
|
||||
wav_path = args.output_dir / (file_name + ".wav")
|
||||
wav_path.write_bytes(wav_bytes)
|
||||
|
||||
_LOGGER.debug("Wrote %s", wav_path)
|
||||
else:
|
||||
# Combine all audio and output to stdout at the end
|
||||
state.all_audio += result.audio_bytes
|
||||
state.sample_rate_hz = result.sample_rate_hz
|
||||
state.sample_width_bytes = result.sample_width_bytes
|
||||
state.num_channels = result.num_channels
|
||||
elif isinstance(result, MarkResult):
|
||||
if state.mark_writer:
|
||||
print(result.name, file=state.mark_writer)
|
||||
except Exception:
|
||||
_LOGGER.exception("Error processing result")
|
||||
except Exception:
|
||||
_LOGGER.exception("process_result")
|
||||
|
||||
|
||||
def process_line(
|
||||
line: str,
|
||||
state: CommandLineInterfaceState,
|
||||
line_id: str = "",
|
||||
):
|
||||
from mimic3_tts import SSMLSpeaker
|
||||
|
||||
assert state.tts is not None
|
||||
assert state.result_queue is not None
|
||||
|
||||
args = state.args
|
||||
assert state.tts is not None
|
||||
|
||||
# TODO: SSML
|
||||
state.tts.begin_utterance()
|
||||
if args.ssml:
|
||||
results = SSMLSpeaker(state.tts).speak(line)
|
||||
else:
|
||||
state.tts.begin_utterance()
|
||||
|
||||
# TODO: text language
|
||||
state.tts.speak_text(line)
|
||||
# TODO: text language
|
||||
state.tts.speak_text(line)
|
||||
|
||||
# TODO: CSV
|
||||
text_id = ""
|
||||
result_idx = 0
|
||||
results = state.tts.end_utterance()
|
||||
|
||||
for result in state.tts.end_utterance():
|
||||
if isinstance(result, AudioResult):
|
||||
if args.raw_stream:
|
||||
assert state.raw_queue is not None
|
||||
state.raw_queue.put(result.audio_bytes)
|
||||
elif args.interactive or args.output_dir:
|
||||
# Convert to WAV audio
|
||||
wav_bytes: typing.Optional[bytes] = None
|
||||
if args.interactive:
|
||||
if not wav_bytes:
|
||||
wav_bytes = result.to_wav_bytes()
|
||||
|
||||
play_wav_bytes(wav_bytes)
|
||||
|
||||
if args.output_dir:
|
||||
if not wav_bytes:
|
||||
wav_bytes = result.to_wav_bytes()
|
||||
|
||||
# Determine file name
|
||||
if args.output_naming == OutputNaming.TEXT:
|
||||
# Use text itself
|
||||
file_name = line.strip().replace(" ", "_")
|
||||
file_name = file_name.translate(
|
||||
str.maketrans("", "", string.punctuation.replace("_", ""))
|
||||
)
|
||||
elif args.output_naming == OutputNaming.TIME:
|
||||
# Use timestamp
|
||||
file_name = str(time.time())
|
||||
elif args.output_naming == OutputNaming.ID:
|
||||
if not text_id:
|
||||
text_id = line_id
|
||||
else:
|
||||
text_id = f"{line_id}_{result_idx + 1}"
|
||||
|
||||
file_name = text_id
|
||||
|
||||
assert file_name, f"No file name for text: {line}"
|
||||
wav_path = args.output_dir / (file_name + ".wav")
|
||||
wav_path.write_bytes(wav_bytes)
|
||||
|
||||
_LOGGER.debug("Wrote %s", wav_path)
|
||||
else:
|
||||
# Combine all audio and output to stdout at the end
|
||||
state.all_audio += result.audio_bytes
|
||||
state.sample_rate_hz = result.sample_rate_hz
|
||||
state.sample_width_bytes = result.sample_width_bytes
|
||||
state.num_channels = result.num_channels
|
||||
|
||||
result_idx += 1
|
||||
elif isinstance(result, MarkResult):
|
||||
if state.mark_writer:
|
||||
print(result.name, file=state.mark_writer)
|
||||
|
||||
# text_id = ""
|
||||
|
||||
# for result_idx, result in enumerate(tts_results):
|
||||
# text = result.text
|
||||
|
||||
# # Write before marks
|
||||
# if result.marks_before and state.mark_writer:
|
||||
# for mark_name in result.marks_before:
|
||||
# print(mark_name, file=state.mark_writer)
|
||||
|
||||
# if args.raw_stream:
|
||||
# assert raw_queue is not None
|
||||
# raw_queue.put(result.audio.tobytes())
|
||||
# elif args.interactive or args.output_dir:
|
||||
# # Convert to WAV audio
|
||||
# with io.BytesIO() as wav_io:
|
||||
# wav_write(wav_io, result.sample_rate, result.audio)
|
||||
# wav_data = wav_io.getvalue()
|
||||
|
||||
# assert wav_data is not None
|
||||
|
||||
# if args.interactive:
|
||||
|
||||
# # Play audio
|
||||
# _LOGGER.debug("Playing audio with play command")
|
||||
# try:
|
||||
# subprocess.run(
|
||||
# play_command,
|
||||
# input=wav_data,
|
||||
# stdout=subprocess.DEVNULL,
|
||||
# stderr=subprocess.DEVNULL,
|
||||
# check=True,
|
||||
# )
|
||||
# except FileNotFoundError:
|
||||
# _LOGGER.error(
|
||||
# "Unable to play audio with command '%s'. set with --play-command or redirect stdout",
|
||||
# args.play_command,
|
||||
# )
|
||||
# with open("output.wav", "wb") as output_file:
|
||||
# output_file.write(wav_data)
|
||||
|
||||
# _LOGGER.warning("stdout not redirected. Wrote audio to output.wav.")
|
||||
|
||||
# else:
|
||||
# # Combine all audio and output to stdout at the end
|
||||
# all_audios.append(result.audio)
|
||||
|
||||
# # Write after marks
|
||||
# if result.marks_after and state.mark_writer:
|
||||
# for mark_name in result.marks_after:
|
||||
# print(mark_name, file=state.mark_writer)
|
||||
for result in results:
|
||||
state.result_queue.put(
|
||||
ResultToProcess(
|
||||
result=result,
|
||||
line=line,
|
||||
line_id=line_id,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def process_lines(state: CommandLineInterfaceState):
|
||||
assert state.texts is not None
|
||||
|
||||
args = state.args
|
||||
start_time_to_first_audio = time.perf_counter()
|
||||
|
||||
try:
|
||||
result_idx = 0
|
||||
|
||||
for line in state.texts:
|
||||
line_id = ""
|
||||
line = line.strip()
|
||||
|
|
@ -386,20 +352,22 @@ def process_lines(state: CommandLineInterfaceState):
|
|||
# Line has the format id|text instead of just text
|
||||
line_id, line = line.split(args.id_delimiter, maxsplit=1)
|
||||
|
||||
process_line(line_id, line, state)
|
||||
process_line(line, state, line_id=line_id)
|
||||
result_idx += 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
if state.raw_queue is not None:
|
||||
if state.result_queue is not None:
|
||||
# Draw audio playback queue
|
||||
while not state.raw_queue.empty():
|
||||
state.raw_queue.get()
|
||||
while not state.result_queue.empty():
|
||||
state.result_queue.get()
|
||||
finally:
|
||||
# Wait for raw stream to finish
|
||||
if state.raw_queue is not None:
|
||||
state.raw_queue.put(None)
|
||||
if state.result_queue is not None:
|
||||
state.result_queue.put(None)
|
||||
|
||||
if state.raw_stream_thread is not None:
|
||||
state.raw_stream_thread.join()
|
||||
if state.result_thread is not None:
|
||||
print("Waiting for audio to finish...", file=sys.stderr)
|
||||
state.result_thread.join()
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
|
|
@ -502,37 +470,24 @@ def get_args():
|
|||
parser.add_argument(
|
||||
"--noise-scale",
|
||||
type=float,
|
||||
default=0.333,
|
||||
help="Noise scale (default: 0.333)",
|
||||
help="Noise scale [0-1], default is 0.667",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--length-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Length scale (default: 1.0)",
|
||||
help="Length scale (1.0 is default speed, 0.5 is 2x faster)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--noise-w",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Variation in cadence (default: 1.0)",
|
||||
help="Variation in cadence [0-1], default is 0.8",
|
||||
)
|
||||
|
||||
# Miscellaneous
|
||||
parser.add_argument(
|
||||
"--max-thread-workers",
|
||||
type=int,
|
||||
help="Maximum number of threads to concurrently load models and run sentences through TTS/Vocoder",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-stream",
|
||||
action="store_true",
|
||||
help="Stream raw 16-bit 22050Hz mono PCM audio to stdout",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-stream-queue-size",
|
||||
"--result-queue-size",
|
||||
default=5,
|
||||
help="Maximum number of sentences to maintain in output queue with --raw-stream (default: 5)",
|
||||
help="Maximum number of sentences to maintain in output queue (default: 5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--process-on-blank-line",
|
||||
|
|
@ -552,6 +507,9 @@ def get_args():
|
|||
action="store_true",
|
||||
help="Force audio output to stdout even if a tty is detected",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preload-voice", action="append", help="Preload voice when starting up"
|
||||
)
|
||||
parser.add_argument("--seed", type=int, help="Set random seed (default: not set)")
|
||||
# parser.add_argument("--version", action="store_true", help="Print version and exit")
|
||||
parser.add_argument(
|
||||
|
|
@ -575,116 +533,6 @@ def get_args():
|
|||
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
# # Directories to search for voices
|
||||
# voices_dirs = get_voices_dirs(args.voices_dir)
|
||||
|
||||
# def list_voices_vocoders():
|
||||
# """Print all vocoders and voices"""
|
||||
# # (type, name) -> location
|
||||
# local_info = {}
|
||||
|
||||
# # Search for downloaded voices/vocoders
|
||||
# for voices_dir in voices_dirs:
|
||||
# if not voices_dir.is_dir():
|
||||
# continue
|
||||
|
||||
# for voice_dir in voices_dir.iterdir():
|
||||
# if not voice_dir.is_dir():
|
||||
# continue
|
||||
|
||||
# if voice_dir.name in VOCODER_DIR_NAMES:
|
||||
# # Vocoder
|
||||
# for vocoder_model_dir in voice_dir.iterdir():
|
||||
# if not valid_voice_dir(vocoder_model_dir):
|
||||
# continue
|
||||
|
||||
# full_vocoder_name = f"{voice_dir.name}-{vocoder_model_dir.name}"
|
||||
# local_info[("vocoder", full_vocoder_name)] = str(
|
||||
# vocoder_model_dir
|
||||
# )
|
||||
# else:
|
||||
# # Voice
|
||||
# voice_lang = voice_dir.name
|
||||
# for voice_model_dir in voice_dir.iterdir():
|
||||
# if not valid_voice_dir(voice_model_dir):
|
||||
# continue
|
||||
|
||||
# local_info[("voice", voice_model_dir.name)] = str(
|
||||
# voice_model_dir
|
||||
# )
|
||||
|
||||
# # (type, lang, name, downloaded, aliases, location)
|
||||
# voices_and_vocoders = []
|
||||
# with open(_DIR / "VOCODERS", "r", encoding="utf-8") as vocoders_file:
|
||||
# for line in vocoders_file:
|
||||
# line = line.strip()
|
||||
# if not line:
|
||||
# continue
|
||||
|
||||
# *vocoder_aliases, full_vocoder_name = line.split()
|
||||
# downloaded = False
|
||||
|
||||
# location = local_info.get(("vocoder", full_vocoder_name), "")
|
||||
# if location:
|
||||
# downloaded = True
|
||||
|
||||
# voices_and_vocoders.append(
|
||||
# (
|
||||
# "vocoder",
|
||||
# " ",
|
||||
# "*" if downloaded else " ",
|
||||
# full_vocoder_name,
|
||||
# ",".join(vocoder_aliases),
|
||||
# location,
|
||||
# )
|
||||
# )
|
||||
|
||||
# with open(_DIR / "VOICES", "r", encoding="utf-8") as voices_file:
|
||||
# for line in voices_file:
|
||||
# line = line.strip()
|
||||
# if not line:
|
||||
# continue
|
||||
|
||||
# *voice_aliases, full_voice_name, download_name = line.split()
|
||||
# voice_lang = download_name.split("_", maxsplit=1)[0]
|
||||
|
||||
# downloaded = False
|
||||
|
||||
# location = local_info.get(("voice", full_voice_name), "")
|
||||
# if location:
|
||||
# downloaded = True
|
||||
|
||||
# voices_and_vocoders.append(
|
||||
# (
|
||||
# "voice",
|
||||
# voice_lang,
|
||||
# "*" if downloaded else " ",
|
||||
# full_voice_name,
|
||||
# ",".join(voice_aliases),
|
||||
# location,
|
||||
# )
|
||||
# )
|
||||
|
||||
# headers = ("TYPE", "LANG", "LOCAL", "NAME", "ALIASES", "LOCATION")
|
||||
|
||||
# # Get widths of columns
|
||||
# col_widths = [0] * len(voices_and_vocoders[0])
|
||||
# for item in voices_and_vocoders:
|
||||
# for col in range(len(col_widths)):
|
||||
# col_widths[col] = max(
|
||||
# col_widths[col], len(item[col]) + 1, len(headers[col]) + 1
|
||||
# )
|
||||
|
||||
# # Print results
|
||||
# print(*(h.ljust(col_widths[col]) for col, h in enumerate(headers)))
|
||||
|
||||
# for item in sorted(voices_and_vocoders):
|
||||
# print(*(v.ljust(col_widths[col]) for col, v in enumerate(item)))
|
||||
|
||||
# if args.list:
|
||||
# list_voices_vocoders()
|
||||
# sys.exit(0)
|
||||
|
||||
return args
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,13 +15,13 @@
|
|||
#
|
||||
import argparse
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import io
|
||||
import logging
|
||||
import sys
|
||||
import io
|
||||
import wave
|
||||
import tempfile
|
||||
import typing
|
||||
import dataclasses
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs
|
||||
|
|
@ -29,6 +29,7 @@ from uuid import uuid4
|
|||
|
||||
import hypercorn
|
||||
import quart_cors
|
||||
from mimic3_tts import AudioResult, Mimic3Settings, Mimic3TextToSpeechSystem
|
||||
from quart import (
|
||||
Quart,
|
||||
Response,
|
||||
|
|
@ -38,8 +39,6 @@ from quart import (
|
|||
send_from_directory,
|
||||
)
|
||||
|
||||
from mimic3_tts import Mimic3TextToSpeechSystem, Mimic3Settings, AudioResult
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
_MISSING = object()
|
||||
|
|
@ -79,6 +78,9 @@ parser.add_argument(
|
|||
default=_MISSING,
|
||||
help="Enable WAV cache with optional directory (default: no cache)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preload-voice", action="append", help="Preload voice when starting up"
|
||||
)
|
||||
# parser.add_argument(
|
||||
# "--max-loaded-models",
|
||||
# type=int,
|
||||
|
|
@ -132,7 +134,6 @@ _WAV_CACHE: typing.Dict[TextToWavParams, Path] = {}
|
|||
|
||||
# TODO: XDG voice directories
|
||||
# TODO: args.voices_dir
|
||||
# TODO: Preload voice
|
||||
|
||||
mimic3 = Mimic3TextToSpeechSystem(
|
||||
Mimic3Settings(
|
||||
|
|
@ -144,6 +145,11 @@ mimic3 = Mimic3TextToSpeechSystem(
|
|||
)
|
||||
)
|
||||
|
||||
if args.preload_voice:
|
||||
for voice_key in args.preload_voice:
|
||||
_LOGGER.debug("Preloading voice: %s", voice_key)
|
||||
mimic3.preload_voice(voice_key)
|
||||
|
||||
|
||||
def text_to_wav(params: TextToWavParams, no_cache: bool = False) -> bytes:
|
||||
|
||||
|
|
|
|||
20
mimic3-http/run.sh
Executable file
20
mimic3-http/run.sh
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# Directory of *this* script
|
||||
this_dir="$( cd "$( dirname "$0" )" && pwd )"
|
||||
|
||||
# Kebab to snake case
|
||||
module_name="$(basename "${this_dir}" | sed -e 's/-/_/g')"
|
||||
src_dir="${this_dir}/${module_name}"
|
||||
|
||||
# Path to virtual environment
|
||||
: "${venv:=${this_dir}/.venv}"
|
||||
|
||||
if [ -d "${venv}" ]; then
|
||||
# Activate virtual environment if available
|
||||
source "${venv}/bin/activate"
|
||||
fi
|
||||
|
||||
export PYTHONPATH="${this_dir}"
|
||||
python3 -m "${module_name}" "$@"
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
from opentts_abc import AudioResult, MarkResult
|
||||
from opentts_abc.ssml import SSMLSpeaker
|
||||
|
||||
from .tts import Mimic3Settings, Mimic3TextToSpeechSystem
|
||||
|
|
|
|||
|
|
@ -26,7 +26,6 @@ from opentts_abc import (
|
|||
)
|
||||
|
||||
from mimic3_tts.config import TrainingConfig
|
||||
from mimic3_tts.utils import audio_float_to_int16
|
||||
from mimic3_tts.voice import Mimic3Voice, SPEAKER_TYPE
|
||||
|
||||
_DIR = Path(__file__).parent
|
||||
|
|
@ -48,9 +47,9 @@ class Mimic3Settings:
|
|||
language: typing.Optional[str] = None
|
||||
voices_directories: typing.Optional[typing.Iterable[typing.Union[str, Path]]] = None
|
||||
speaker: typing.Optional[SPEAKER_TYPE] = None
|
||||
length_scale: float = 1.0
|
||||
noise_scale: float = 0.667
|
||||
noise_w: float = 0.8
|
||||
length_scale: typing.Optional[float] = None
|
||||
noise_scale: typing.Optional[float] = None
|
||||
noise_w: typing.Optional[float] = None
|
||||
text_language: typing.Optional[str] = None
|
||||
sample_rate: int = 22050
|
||||
|
||||
|
|
@ -59,6 +58,12 @@ class Mimic3Settings:
|
|||
class Mimic3Phonemes:
|
||||
current_settings: Mimic3Settings
|
||||
phonemes: typing.List[typing.List[str]] = field(default_factory=list)
|
||||
is_utterance: bool = True
|
||||
|
||||
|
||||
class VoiceNotFoundError(Exception):
|
||||
def __init__(self, voice: str):
|
||||
super().__init__(f"Voice not found: {voice}")
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -71,8 +76,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
self.settings = settings
|
||||
|
||||
self._results: typing.List[typing.Union[BaseResult, Mimic3Phonemes]] = []
|
||||
|
||||
self.loaded_voices: typing.Dict[str, Mimic3Voice] = {}
|
||||
self._loaded_voices: typing.Dict[str, Mimic3Voice] = {}
|
||||
|
||||
@property
|
||||
def voice(self) -> str:
|
||||
|
|
@ -173,7 +177,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
)
|
||||
|
||||
def begin_utterance(self):
|
||||
self._results.clear()
|
||||
pass
|
||||
|
||||
def speak_text(self, text: str, text_language: typing.Optional[str] = None):
|
||||
voice = self._get_or_load_voice(self.voice)
|
||||
|
|
@ -275,7 +279,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
|
||||
for result in self._results:
|
||||
if isinstance(result, Mimic3Phonemes):
|
||||
if result.current_settings != last_settings:
|
||||
if result.is_utterance or (result.current_settings != last_settings):
|
||||
if sent_phonemes:
|
||||
yield self._speak_sentence_phonemes(
|
||||
sent_phonemes, settings=last_settings
|
||||
|
|
@ -295,9 +299,15 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
|
||||
if sent_phonemes:
|
||||
yield self._speak_sentence_phonemes(sent_phonemes)
|
||||
sent_phonemes.clear()
|
||||
|
||||
self._results.clear()
|
||||
|
||||
def preload_voice(self, voice_key: str):
|
||||
self._get_or_load_voice(voice_key)
|
||||
|
||||
def _get_or_load_voice(self, voice_key: str) -> Mimic3Voice:
|
||||
existing_voice = self.loaded_voices.get(voice_key)
|
||||
existing_voice = self._loaded_voices.get(voice_key)
|
||||
if existing_voice is not None:
|
||||
return existing_voice
|
||||
|
||||
|
|
@ -308,11 +318,16 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
model_dir = Path(maybe_voice.key)
|
||||
break
|
||||
|
||||
assert model_dir is not None
|
||||
existing_voice = self.loaded_voices.get(str(model_dir.absolute()))
|
||||
if model_dir is None:
|
||||
raise VoiceNotFoundError(voice_key)
|
||||
|
||||
# Full path to voice model directory
|
||||
canonical_key = str(model_dir.absolute())
|
||||
|
||||
existing_voice = self._loaded_voices.get(canonical_key)
|
||||
if existing_voice is not None:
|
||||
# Alias
|
||||
self.loaded_voices[voice_key] = existing_voice
|
||||
self._loaded_voices[voice_key] = existing_voice
|
||||
|
||||
return existing_voice
|
||||
|
||||
|
|
@ -321,355 +336,7 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
|
|||
_LOGGER.info("Loaded voice from %s", model_dir)
|
||||
|
||||
# Add to cache
|
||||
self.loaded_voices[voice_key] = voice
|
||||
self._loaded_voices[voice_key] = voice
|
||||
self._loaded_voices[canonical_key] = voice
|
||||
|
||||
return voice
|
||||
|
||||
# def start(self):
|
||||
# self.stop()
|
||||
|
||||
# self._thread = threading.Thread(target=self._thread_proc, daemon=True)
|
||||
# self._thread.start()
|
||||
|
||||
# def stop(self):
|
||||
# if self._thread is not None:
|
||||
# self._request_queue.put(None)
|
||||
# self._thread.join()
|
||||
# self._thread = None
|
||||
|
||||
# # Drain queues
|
||||
# while not self._request_queue.empty():
|
||||
# self._request_queue.get()
|
||||
|
||||
# while not self._result_queue.empty():
|
||||
# self._result_queue.get()
|
||||
|
||||
# def _thread_proc(self):
|
||||
# try:
|
||||
# self._load_model()
|
||||
# self._load_text_processor()
|
||||
|
||||
# while True:
|
||||
# message = self._request_queue.get()
|
||||
# if message is None:
|
||||
# break
|
||||
|
||||
# if isinstance(message, AddLexiconMessage):
|
||||
# self._add_lexicon(message.lexicon_file)
|
||||
# elif isinstance(message, TextToSpeechMessage):
|
||||
# result = self._text_to_speech(**dataclasses.asdict(message))
|
||||
# self._result_queue.put(result)
|
||||
|
||||
# except Exception:
|
||||
# _LOGGER.exception("_thread_proc")
|
||||
|
||||
# def _load_model(self):
|
||||
# """Load model configuration and generator"""
|
||||
|
||||
# if self._config is None:
|
||||
# config_path = self.model_dir / "config.json"
|
||||
# _LOGGER.debug("Loading model config from %s", config_path)
|
||||
|
||||
# with open(config_path, "r", encoding="utf-8") as config_file:
|
||||
# self._config = TrainingConfig.load(config_file)
|
||||
|
||||
# self.lang = self.lang or self._config.text_language or "en_US"
|
||||
|
||||
# if self._phoneme_to_id is None:
|
||||
# # phoneme -> id
|
||||
# phoneme_ids_path = self.model_dir / "phonemes.txt"
|
||||
# _LOGGER.debug("Loading model phonemes from %s", phoneme_ids_path)
|
||||
# with open(phoneme_ids_path, "r", encoding="utf-8") as ids_file:
|
||||
# self._phoneme_to_id = phonemes2ids.load_phoneme_ids(ids_file)
|
||||
|
||||
# valid_phonemes = []
|
||||
# for phoneme_str in self._phoneme_to_id:
|
||||
# maybe_phoneme = Phoneme(phoneme_str)
|
||||
# if any(
|
||||
# [
|
||||
# maybe_phoneme.vowel,
|
||||
# maybe_phoneme.consonant,
|
||||
# maybe_phoneme.dipthong,
|
||||
# maybe_phoneme.schwa,
|
||||
# ]
|
||||
# ):
|
||||
# valid_phonemes.append(maybe_phoneme)
|
||||
|
||||
# self._voice_phonemes = Phonemes(phonemes=valid_phonemes)
|
||||
|
||||
# if self._phoneme_map is None:
|
||||
# # phoneme -> phoneme, phoneme, ...
|
||||
# phoneme_map_path = self.model_dir / "phoneme_map.txt"
|
||||
# if phoneme_map_path.is_file():
|
||||
# _LOGGER.debug("Loading phoneme map from %s", phoneme_map_path)
|
||||
# with open(phoneme_map_path, "r", encoding="utf-8") as map_file:
|
||||
# self._phoneme_map = phonemes2ids.utils.load_phoneme_map(map_file)
|
||||
|
||||
# if self._onnx_model is None:
|
||||
# generator_path = self.model_dir / "generator.onnx"
|
||||
# _LOGGER.debug("Loading model from %s", generator_path)
|
||||
|
||||
# sess_options = onnxruntime.SessionOptions()
|
||||
# sess_options.enable_cpu_mem_arena = False
|
||||
# sess_options.enable_mem_pattern = False
|
||||
# sess_options.enable_mem_reuse = False
|
||||
|
||||
# self._onnx_model = onnxruntime.InferenceSession(
|
||||
# str(generator_path), sess_options=sess_options
|
||||
# )
|
||||
|
||||
# def _load_text_processor(self):
|
||||
# if self._text_processor is None:
|
||||
# self._text_processor = gruut.TextProcessor(default_lang=self.lang)
|
||||
|
||||
# def add_lexicon(self, lexicon_file: typing.Iterable[str]):
|
||||
# """Load a custom pronunciation lexicon from a file.
|
||||
|
||||
# Format is:
|
||||
# <word> <role> <phoneme> <phoneme> ...
|
||||
|
||||
# Role can be things like "gruut:VB" or "gruut:NN".
|
||||
# Use "_" for the default role (any part of speech).
|
||||
# """
|
||||
# self._request_queue.put(AddLexiconMessage(lexicon_file=list(lexicon_file)))
|
||||
|
||||
# def _add_lexicon(self, lexicon_file: typing.Iterable[str]):
|
||||
# self._load_text_processor()
|
||||
# assert self._text_processor is not None
|
||||
|
||||
# # word -> role -> [phoneme, phoneme, ...]
|
||||
# lexicon: typing.Dict[str, typing.Dict[str, typing.List[str]]] = {}
|
||||
|
||||
# for line in lexicon_file:
|
||||
# line = line.strip()
|
||||
# if not line:
|
||||
# continue
|
||||
|
||||
# word, role, *phonemes = line.split()
|
||||
# if (not word) or (not phonemes):
|
||||
# _LOGGER.warning("Empty word or pronunciation in lexicon: %s", line)
|
||||
# continue
|
||||
|
||||
# if role == "_":
|
||||
# role = WordRole.DEFAULT
|
||||
|
||||
# word_roles = lexicon.get(word)
|
||||
# if word_roles is None:
|
||||
# word_roles = {}
|
||||
# lexicon[word] = word_roles
|
||||
|
||||
# word_roles[role] = phonemes
|
||||
|
||||
# if lexicon:
|
||||
|
||||
# # Wrap the "lookup_phonemes" method in the gruut text processor.
|
||||
# # Our lexicon will be consulted first.
|
||||
# settings = self._text_processor.get_settings()
|
||||
# base_lookup = settings.lookup_phonemes
|
||||
|
||||
# def lookup_phonemes(word: str, role: typing.Optional[str] = None, **kwargs):
|
||||
# word_roles = lexicon.get(word)
|
||||
|
||||
# if not word_roles:
|
||||
# # Try lower case
|
||||
# word_roles = lexicon.get(word.lower())
|
||||
|
||||
# if word_roles:
|
||||
# if role is None:
|
||||
# role = WordRole.DEFAULT
|
||||
|
||||
# phonemes = word_roles.get(role)
|
||||
|
||||
# if (phonemes is None) and (role != WordRole.DEFAULT):
|
||||
# phonemes = word_roles.get(WordRole.DEFAULT)
|
||||
|
||||
# if phonemes:
|
||||
# return phonemes
|
||||
|
||||
# if base_lookup is not None:
|
||||
# return base_lookup(word, role, **kwargs)
|
||||
|
||||
# return None
|
||||
|
||||
# settings.lookup_phonemes = typing.cast(LookupPhonemes, lookup_phonemes)
|
||||
# _LOGGER.debug("Added custom pronunciations for %s word(s)", len(lexicon))
|
||||
|
||||
# def text_to_speech(
|
||||
# self,
|
||||
# text: str,
|
||||
# speaker_id: typing.Optional[int] = None,
|
||||
# length_scale: typing.Optional[float] = None,
|
||||
# noise_scale: typing.Optional[float] = None,
|
||||
# noise_w: typing.Optional[float] = None,
|
||||
# ssml: bool = False,
|
||||
# text_language: typing.Optional[str] = None,
|
||||
# ) -> Result:
|
||||
# self._request_queue.put(
|
||||
# TextToSpeechMessage(
|
||||
# text=text,
|
||||
# speaker_id=speaker_id,
|
||||
# length_scale=length_scale,
|
||||
# noise_scale=noise_scale,
|
||||
# noise_w=noise_w,
|
||||
# ssml=ssml,
|
||||
# text_language=text_language,
|
||||
# )
|
||||
# )
|
||||
|
||||
# result = typing.cast(Result, self._result_queue.get())
|
||||
|
||||
# return result
|
||||
|
||||
# def _text_to_speech(
|
||||
# self,
|
||||
# text: str,
|
||||
# speaker_id: typing.Optional[int] = None,
|
||||
# length_scale: typing.Optional[float] = None,
|
||||
# noise_scale: typing.Optional[float] = None,
|
||||
# noise_w: typing.Optional[float] = None,
|
||||
# ssml: bool = False,
|
||||
# text_language: typing.Optional[str] = None,
|
||||
# ) -> Result:
|
||||
# """Speak text and return WAV audio as bytes"""
|
||||
# text_language = text_language or self.lang
|
||||
# assert self._text_processor is not None
|
||||
|
||||
# # Ensure model is loaded
|
||||
# assert self.lang is not None
|
||||
# assert self._config is not None
|
||||
# assert self._phoneme_to_id is not None
|
||||
# assert self._onnx_model is not None
|
||||
|
||||
# # Resolve settings
|
||||
# if speaker_id is None:
|
||||
# speaker_id = self.speaker_id or 0
|
||||
|
||||
# if length_scale is None:
|
||||
# length_scale = self.length_scale
|
||||
|
||||
# if noise_scale is None:
|
||||
# noise_scale = self.noise_scale
|
||||
|
||||
# if noise_w is None:
|
||||
# noise_w = self.noise_w
|
||||
|
||||
# # Process text into sentences
|
||||
# result = Result(text=text)
|
||||
# audio_arrays: typing.List[np.ndarray] = []
|
||||
|
||||
# graph, root = self._text_processor.process(text, lang=text_language, ssml=ssml)
|
||||
# sentences = list(self._text_processor.sentences(graph, root))
|
||||
|
||||
# for sentence in sentences:
|
||||
# result.sentence_words.append([w.text for w in sentence])
|
||||
|
||||
# if text_language == self.lang:
|
||||
# sent_phonemes = [w.phonemes for w in sentence if w.phonemes]
|
||||
# else:
|
||||
# # Convert phonemes to ids to target language
|
||||
# other_sent_phonemes = [w.phonemes for w in sentence if w.phonemes]
|
||||
# _LOGGER.debug(other_sent_phonemes)
|
||||
|
||||
# sent_phonemes = []
|
||||
# for other_word_p in other_sent_phonemes:
|
||||
# word_p = []
|
||||
# for other_p in other_word_p:
|
||||
# if IPA.is_break(other_p):
|
||||
# # Keep breaks
|
||||
# word_p.append(other_p)
|
||||
# continue
|
||||
|
||||
# original_p = other_p
|
||||
# stress = ""
|
||||
# while other_p and IPA.is_stress(other_p[0]):
|
||||
# stress = other_p[0]
|
||||
# other_p = other_p[1:]
|
||||
|
||||
# if not other_p:
|
||||
# continue
|
||||
|
||||
# if other_p in self._phoneme_to_id:
|
||||
# word_p.append(original_p)
|
||||
# continue
|
||||
|
||||
# assert self._voice_phonemes is not None
|
||||
# guessed = guess_phonemes(
|
||||
# other_p, to_phonemes=self._voice_phonemes
|
||||
# )
|
||||
# if guessed.phonemes:
|
||||
# word_p.extend([p.text for p in guessed.phonemes])
|
||||
|
||||
# if word_p:
|
||||
# sent_phonemes.append(word_p)
|
||||
|
||||
# result.sentence_phonemes.append(sent_phonemes)
|
||||
|
||||
# sent_phoneme_ids = phonemes2ids.phonemes2ids(
|
||||
# word_phonemes=sent_phonemes,
|
||||
# phoneme_to_id=self._phoneme_to_id,
|
||||
# pad=self._config.phonemes.pad,
|
||||
# bos=self._config.phonemes.bos,
|
||||
# eos=self._config.phonemes.eos,
|
||||
# auto_bos_eos=self._config.phonemes.auto_bos_eos,
|
||||
# blank=self._config.phonemes.blank,
|
||||
# blank_word=self._config.phonemes.blank_word,
|
||||
# blank_between=self._config.phonemes.blank_between,
|
||||
# blank_at_start=self._config.phonemes.blank_at_start,
|
||||
# blank_at_end=self._config.phonemes.blank_at_end,
|
||||
# simple_punctuation=self._config.phonemes.simple_punctuation,
|
||||
# punctuation_map=self._config.phonemes.punctuation_map,
|
||||
# separate=self._config.phonemes.separate,
|
||||
# separate_graphemes=self._config.phonemes.separate_graphemes,
|
||||
# separate_tones=self._config.phonemes.separate_tones,
|
||||
# tone_before=self._config.phonemes.tone_before,
|
||||
# phoneme_map=self._phoneme_map or self._config.phonemes.phoneme_map,
|
||||
# fail_on_missing=False,
|
||||
# )
|
||||
|
||||
# result.sentence_phoneme_ids.append(sent_phonemes)
|
||||
|
||||
# _LOGGER.debug("%s %s %s", sentence.text, sent_phonemes, sent_phoneme_ids)
|
||||
|
||||
# # Create model inputs
|
||||
# text_array = np.expand_dims(np.array(sent_phoneme_ids, dtype=np.int64), 0)
|
||||
# text_lengths_array = np.array([text_array.shape[1]], dtype=np.int64)
|
||||
# scales_array = np.array(
|
||||
# [noise_scale, length_scale, noise_w], dtype=np.float32
|
||||
# )
|
||||
|
||||
# inputs = {
|
||||
# "input": text_array,
|
||||
# "input_lengths": text_lengths_array,
|
||||
# "scales": scales_array,
|
||||
# }
|
||||
|
||||
# if self._config.is_multispeaker:
|
||||
# speaker_id_array = np.array([speaker_id], dtype=np.int64)
|
||||
# inputs["sid"] = speaker_id_array
|
||||
|
||||
# # Infer audio from phonemes
|
||||
# start_time = time.perf_counter()
|
||||
# audio = self._onnx_model.run(None, inputs)[0].squeeze()
|
||||
# audio = audio_float_to_int16(audio)
|
||||
# end_time = time.perf_counter()
|
||||
|
||||
# # Compute real-time factor
|
||||
# audio_duration_sec = audio.shape[-1] / self._config.audio.sample_rate
|
||||
# infer_sec = end_time - start_time
|
||||
# real_time_factor = (
|
||||
# infer_sec / audio_duration_sec if audio_duration_sec > 0 else 0.0
|
||||
# )
|
||||
|
||||
# _LOGGER.debug("RTF: %s", real_time_factor)
|
||||
|
||||
# audio_arrays.append(audio)
|
||||
|
||||
# # Write to WAV and return bytes
|
||||
# with io.BytesIO() as wav_file:
|
||||
# write_wav(
|
||||
# wav_file, self._config.audio.sample_rate, np.concatenate(audio_arrays),
|
||||
# )
|
||||
|
||||
# result.wav_bytes = wav_file.getvalue()
|
||||
|
||||
# return result
|
||||
|
|
|
|||
|
|
@ -117,15 +117,29 @@ class Mimic3Voice(metaclass=ABCMeta):
|
|||
speaker: typing.Optional[
|
||||
typing.Union[SPEAKER_NAME_TYPE, SPEAKER_ID_TYPE]
|
||||
] = None,
|
||||
length_scale: float = 1.0,
|
||||
noise_scale: float = 0.333,
|
||||
noise_w: float = 1.0,
|
||||
length_scale: typing.Optional[float] = None,
|
||||
noise_scale: typing.Optional[float] = None,
|
||||
noise_w: typing.Optional[float] = None,
|
||||
) -> np.ndarray:
|
||||
if length_scale is None:
|
||||
length_scale = self.config.inference.length_scale
|
||||
|
||||
if noise_scale is None:
|
||||
noise_scale = self.config.inference.noise_scale
|
||||
|
||||
if noise_w is None:
|
||||
noise_w = self.config.inference.noise_w
|
||||
|
||||
# Create model inputs
|
||||
text_array = np.expand_dims(np.array(phoneme_ids, dtype=np.int64), 0)
|
||||
text_lengths_array = np.array([text_array.shape[1]], dtype=np.int64)
|
||||
scales_array = np.array(
|
||||
[noise_scale, length_scale, noise_w,], dtype=np.float32,
|
||||
[
|
||||
noise_scale,
|
||||
length_scale,
|
||||
noise_w,
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
# TODO: Use settings from voice config
|
||||
|
|
@ -135,8 +149,8 @@ class Mimic3Voice(metaclass=ABCMeta):
|
|||
"scales": scales_array,
|
||||
}
|
||||
|
||||
speaker_id = 0
|
||||
if self.config.is_multispeaker:
|
||||
speaker_id = 0
|
||||
if isinstance(speaker, SPEAKER_NAME_TYPE):
|
||||
if self.speaker_map:
|
||||
maybe_speaker_id = self.speaker_map.get(speaker)
|
||||
|
|
@ -158,6 +172,14 @@ class Mimic3Voice(metaclass=ABCMeta):
|
|||
speaker_id_array = np.array([speaker_id], dtype=np.int64)
|
||||
inputs["sid"] = speaker_id_array
|
||||
|
||||
_LOGGER.debug(
|
||||
"TTS settings: speaker-id=%s, length-scale=%s, noise-scale=%s, noise-w=%s",
|
||||
speaker_id,
|
||||
length_scale,
|
||||
noise_scale,
|
||||
noise_w,
|
||||
)
|
||||
|
||||
# Infer audio from phonemes
|
||||
start_time = time.perf_counter()
|
||||
audio = self.onnx_model.run(None, inputs)[0].squeeze()
|
||||
|
|
|
|||
|
|
@ -111,6 +111,8 @@ class SSMLSpeaker:
|
|||
pass
|
||||
elif end_tag in {"metadata", "meta"}:
|
||||
self.handle_end_metadata()
|
||||
elif end_tag == "speak":
|
||||
yield from self.handle_end_speak()
|
||||
else:
|
||||
LOG.debug("Ignoring end tag: %s", end_tag)
|
||||
else:
|
||||
|
|
@ -278,6 +280,12 @@ class SSMLSpeaker:
|
|||
|
||||
yield from self.tts.end_utterance()
|
||||
|
||||
def handle_end_speak(self) -> typing.Iterable[BaseResult]:
|
||||
LOG.debug("end speak")
|
||||
assert self.state in {ParsingState.DEFAULT}, self.state
|
||||
|
||||
yield from self.tts.end_utterance()
|
||||
|
||||
def handle_begin_voice(self, elem: etree.Element):
|
||||
LOG.debug("begin voice")
|
||||
voice_name = attrib_no_namespace(elem, "name")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue