From b1d75d7ff606d9304192d61911d16ccfbbc79bb5 Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Wed, 30 Mar 2022 17:08:56 -0400 Subject: [PATCH] Working single-file binary build --- .dockerignore | 3 + Dockerfile | 15 +-- Dockerfile.binary | 116 ++++++++++++++++++ Makefile | 10 +- mimic3-tts/check.sh | 3 +- mimic3-tts/mimic3_tts/__main__.py | 29 +++-- mimic3-tts/mimic3_tts/const.py | 4 +- mimic3-tts/mypy.ini | 3 - mimic3-tts/requirements.txt | 3 +- mimic3-tts/requirements_dev.txt | 7 -- mimic3-tts/setup.py | 2 +- opentts-abc/install.sh | 34 ----- opentts-abc/requirements_dev.txt | 7 -- pyinstaller/mimic3.py | 21 ++++ pyinstaller/mimic3_server.py | 21 ++++ ...quirements_dev.txt => requirements_dev.txt | 2 +- 16 files changed, 202 insertions(+), 78 deletions(-) create mode 100644 Dockerfile.binary delete mode 100644 mimic3-tts/requirements_dev.txt delete mode 100755 opentts-abc/install.sh delete mode 100644 opentts-abc/requirements_dev.txt create mode 100644 pyinstaller/mimic3.py create mode 100644 pyinstaller/mimic3_server.py rename mimic3-http/requirements_dev.txt => requirements_dev.txt (86%) diff --git a/.dockerignore b/.dockerignore index 05c0b4d..c537b96 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ * !LICENSE +!install.sh +!pyinstaller/*.py # opentts-abc !opentts-abc/setup.py @@ -14,6 +16,7 @@ !mimic3-tts/MANIFEST.in !mimic3-tts/requirements.txt !mimic3-tts/mimic3_tts/*.py +!mimic3-tts/mimic3_tts/voices.json !mimic3-tts/mimic3_tts/VERSION # HTTP server diff --git a/Dockerfile b/Dockerfile index f5963f5..7cddb4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,22 +38,13 @@ RUN --mount=type=cache,id=apt-build,target=/var/cache/apt \ python3 python3-pip python3-venv \ build-essential python3-dev -RUN mkdir -p /home/mimic3/app WORKDIR /home/mimic3/app -# Create virtual environment -RUN --mount=type=cache,id=pip-venv,target=/root/.cache/pip \ - python3 -m venv .venv && \ - .venv/bin/pip3 install --upgrade pip && \ - .venv/bin/pip3 install --upgrade wheel setuptools +COPY ./ ./ -COPY ./ /home/mimic3/app/ - -# Install packages in order +# Install mimic3 RUN --mount=type=cache,id=pip-requirements,target=/root/.cache/pip \ - .venv/bin/pip3 install -e opentts-abc && \ - .venv/bin/pip3 install -e mimic3-tts && \ - .venv/bin/pip3 install -e mimic3-http + ./install.sh # ----------------------------------------------------------------------------- diff --git a/Dockerfile.binary b/Dockerfile.binary new file mode 100644 index 0000000..5914cb8 --- /dev/null +++ b/Dockerfile.binary @@ -0,0 +1,116 @@ +# Copyright 2022 Mycroft AI Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# ----------------------------------------------------------------------------- +# Dockerfile for Mimic 3 (https://github.com/MycroftAI/mimic3) +# +# Creates self-contained binaries using PyInstaller +# (https://pyinstaller.readthedocs.io/en/stable/). +# +# Requires Docker buildx: https://docs.docker.com/buildx/working-with-buildx/ +# ----------------------------------------------------------------------------- + +FROM debian:bullseye as build +ARG TARGETARCH +ARG TARGETVARIANT + +ENV LANG C.UTF-8 +ENV DEBIAN_FRONTEND=noninteractive + +RUN echo "Dir::Cache var/cache/apt/${TARGETARCH}${TARGETVARIANT};" > /etc/apt/apt.conf.d/01cache + +RUN --mount=type=cache,id=apt-build,target=/var/cache/apt \ + mkdir -p /var/cache/apt/${TARGETARCH}${TARGETVARIANT}/archives/partial && \ + apt-get update && \ + apt-get install --yes --no-install-recommends \ + python3 python3-pip python3-venv \ + build-essential python3-dev \ + libespeak-ng1 + +WORKDIR /build/mimic3 +COPY ./ ./ + +# Install mimic3 +RUN --mount=type=cache,id=pip-venv,target=/root/.cache/pip \ + ./install.sh + +# Install PyInstaller +RUN --mount=type=cache,id=pip-venv,target=/root/.cache/pip \ + .venv/bin/pip3 install 'pyinstaller>=4,<5' + +# Create binary for command-line interface +RUN .venv/bin/pyinstaller \ + --onefile \ + --noconfirm \ + --name mimic3 \ + --hidden-import 'pycrfsuite._dumpparser' \ + --hidden-import 'pycrfsuite._logparser' \ + --collect-binaries 'onnxruntime' \ + --add-binary "$(/sbin/ldconfig -p | grep libespeak-ng.so.1 | awk '{print $4}'):." \ + --collect-data 'gruut' \ + --hidden-import "gruut_lang_de" \ + --collect-data "gruut_lang_de" \ + --hidden-import "gruut_lang_en" \ + --collect-data "gruut_lang_en" \ + --hidden-import "gruut_lang_fr" \ + --collect-data "gruut_lang_fr" \ + --hidden-import "gruut_lang_it" \ + --collect-data "gruut_lang_it" \ + --hidden-import "gruut_lang_nl" \ + --collect-data "gruut_lang_nl" \ + --hidden-import "gruut_lang_ru" \ + --collect-data "gruut_lang_ru" \ + --hidden-import "gruut_lang_sw" \ + --collect-data "gruut_lang_sw" \ + --collect-data 'espeak_phonemizer' \ + --collect-data 'phonemes2ids' \ + --collect-data 'mimic3_tts' \ + pyinstaller/mimic3.py + +# Create binary for web server +RUN .venv/bin/pyinstaller \ + --onefile \ + --noconfirm \ + --name mimic3-server \ + --hidden-import 'pycrfsuite._dumpparser' \ + --hidden-import 'pycrfsuite._logparser' \ + --collect-binaries 'onnxruntime' \ + --add-binary "$(/sbin/ldconfig -p | grep libespeak-ng.so.1 | awk '{print $4}'):." \ + --collect-data 'gruut' \ + --hidden-import "gruut_lang_de" \ + --collect-data "gruut_lang_de" \ + --hidden-import "gruut_lang_en" \ + --collect-data "gruut_lang_en" \ + --hidden-import "gruut_lang_fr" \ + --collect-data "gruut_lang_fr" \ + --hidden-import "gruut_lang_it" \ + --collect-data "gruut_lang_it" \ + --hidden-import "gruut_lang_nl" \ + --collect-data "gruut_lang_nl" \ + --hidden-import "gruut_lang_ru" \ + --collect-data "gruut_lang_ru" \ + --hidden-import "gruut_lang_sw" \ + --collect-data "gruut_lang_sw" \ + --collect-data 'espeak_phonemizer' \ + --collect-data 'phonemes2ids' \ + --collect-data 'mimic3_tts' \ + --collect-data 'mimic3_http' \ + pyinstaller/mimic3_server.py + +# ----------------------------------------------------------------------------- + +FROM scratch + +COPY --from=build /build/mimic3/dist/ / diff --git a/Makefile b/Makefile index c16d751..bc92473 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # -.PHONY: dist install docker +.PHONY: dist install docker binaries + +SHELL := bash +DOCKER_PLATFORM := linux/amd64 dist: cd opentts-abc && python3 setup.py sdist @@ -28,4 +31,7 @@ install: ./install.sh docker: - docker buildx build . -f Dockerfile --tag mycroftai/mimic3 --load + docker buildx build . -f Dockerfile --platform $(DOCKER_PLATFORM) --tag mycroftai/mimic3 --load + +binaries: + docker buildx build . -f Dockerfile.binary $(DOCKER_PLATFORM) --output type=local,dest=dist/$(DOCKER_PLATFORM) diff --git a/mimic3-tts/check.sh b/mimic3-tts/check.sh index 6c87eea..167e640 100755 --- a/mimic3-tts/check.sh +++ b/mimic3-tts/check.sh @@ -6,10 +6,11 @@ this_dir="$( cd "$( dirname "$0" )" && pwd )" # Kebab to snake case module_name="$(basename "${this_dir}" | sed -e 's/-/_/g')" +base_dir="$(realpath "${this_dir}/..")" src_dir="${this_dir}/${module_name}" # Path to virtual environment -: "${venv:=${this_dir}/.venv}" +: "${venv:=${base_dir}/.venv}" if [ -d "${venv}" ]; then # Activate virtual environment if available diff --git a/mimic3-tts/mimic3_tts/__main__.py b/mimic3-tts/mimic3_tts/__main__.py index 0f12df8..100beff 100644 --- a/mimic3-tts/mimic3_tts/__main__.py +++ b/mimic3-tts/mimic3_tts/__main__.py @@ -19,7 +19,10 @@ import csv import io import logging import os +import shlex +import shutil import string +import subprocess import sys import tempfile import threading @@ -39,6 +42,7 @@ if typing.TYPE_CHECKING: _LOGGER = logging.getLogger(_PACKAGE) +_DEFAULT_PLAY_PROGRAMS = ["paplay", "play -q", "aplay -q"] # ----------------------------------------------------------------------------- @@ -264,7 +268,7 @@ def process_result(state: CommandLineInterfaceState): wav_bytes = result.to_wav_bytes() if wav_bytes: - play_wav_bytes(wav_bytes) + play_wav_bytes(state.args, wav_bytes) if args.output_dir: if not wav_bytes: @@ -387,7 +391,7 @@ def process_lines(state: CommandLineInterfaceState): wav_file_play.setnchannels(state.num_channels) wav_file_play.writeframes(state.all_audio) - play_wav_bytes(wav_io.getvalue()) + play_wav_bytes(state.args, wav_io.getvalue()) else: # Write output directly to stdout wav_file_write: wave.Wave_write = wave.open(sys.stdout.buffer, "wb") @@ -406,15 +410,20 @@ def shutdown_tts(state: CommandLineInterfaceState): state.tts = None -def play_wav_bytes(wav_bytes: bytes): - from playsound import playsound - +def play_wav_bytes(args: argparse.Namespace, wav_bytes: bytes): with tempfile.NamedTemporaryFile(mode="wb+", suffix=".wav") as wav_file: wav_file.write(wav_bytes) wav_file.seek(0) - _LOGGER.debug("Playing WAV file: %s", wav_file.name) - playsound(wav_file.name) + for play_program in reversed(args.play_program): + play_cmd = shlex.split(play_program) + if not shutil.which(play_cmd[0]): + continue + + play_cmd.append(wav_file.name) + _LOGGER.debug("Playing WAV file: %s", play_cmd) + subprocess.check_output(play_cmd) + break def print_voices(state: CommandLineInterfaceState): @@ -521,6 +530,12 @@ def get_args(): parser.add_argument( "--preload-voice", action="append", help="Preload voice when starting up" ) + parser.add_argument( + "--play-program", + action="append", + default=_DEFAULT_PLAY_PROGRAMS, + help="Program(s) used to play WAV files", + ) parser.add_argument("--seed", type=int, help="Set random seed (default: not set)") parser.add_argument("--version", action="store_true", help="Print version and exit") parser.add_argument( diff --git a/mimic3-tts/mimic3_tts/const.py b/mimic3-tts/mimic3_tts/const.py index 0319524..b7b27a1 100644 --- a/mimic3-tts/mimic3_tts/const.py +++ b/mimic3-tts/mimic3_tts/const.py @@ -19,5 +19,7 @@ from xdgenvpy import XDG DEFAULT_VOICE = "en_US/vctk_low" DEFAULT_LANGUAGE = "en_US" -DEFAULT_VOICES_URL_FORMAT = "https://github.com/MycroftAI/mimic3-voices/raw/master/voices/{lang}/{name}" +DEFAULT_VOICES_URL_FORMAT = ( + "https://github.com/MycroftAI/mimic3-voices/raw/master/voices/{lang}/{name}" +) DEFAULT_VOICES_DOWNLOAD_DIR = Path(XDG().XDG_DATA_HOME) / "mimic3" / "voices" diff --git a/mimic3-tts/mypy.ini b/mimic3-tts/mypy.ini index c002909..bbe7a0e 100644 --- a/mimic3-tts/mypy.ini +++ b/mimic3-tts/mypy.ini @@ -6,9 +6,6 @@ ignore_missing_imports = True [mypy-onnxruntime.*] ignore_missing_imports = True -[mypy-playsound.*] -ignore_missing_imports = True - [mypy-tqdm.*] ignore_missing_imports = True diff --git a/mimic3-tts/requirements.txt b/mimic3-tts/requirements.txt index 3cb3c4a..14f2ee1 100644 --- a/mimic3-tts/requirements.txt +++ b/mimic3-tts/requirements.txt @@ -1,10 +1,9 @@ dataclasses-json<1.0 espeak-phonemizer>=1.0,<2.0 -gruut>=2.2.2,<3.0 +gruut>=2.3.0,<3.0 numpy<2.0 onnxruntime>=1.6,<2.0 opentts_abc<1.0 phonemes2ids<2.0 -playsound~=1.3.0 tqdm>=4,<5 xdgenvpy>2.0,<3 diff --git a/mimic3-tts/requirements_dev.txt b/mimic3-tts/requirements_dev.txt deleted file mode 100644 index 5318af0..0000000 --- a/mimic3-tts/requirements_dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -black==22.1.0 -coverage==5.0.4 -flake8==3.7.9 -mypy==0.910 -pylint==2.10.2 -pytest==5.4.1 -pytest-cov==2.8.1 diff --git a/mimic3-tts/setup.py b/mimic3-tts/setup.py index c478920..95db292 100644 --- a/mimic3-tts/setup.py +++ b/mimic3-tts/setup.py @@ -81,7 +81,7 @@ setup( author_email="michael.hansen@mycroft.ai", license="AGPLv3+", packages=setuptools.find_packages(), - package_data={"mimic3_tts": ["VERSION", "py.typed"]}, + package_data={"mimic3_tts": ["VERSION", "py.typed", "voices.json"]}, install_requires=requirements, extras_require={':python_version<"3.9"': ["importlib_resources"], **extras_require}, entry_points={"console_scripts": ["mimic3 = mimic3_cli.__main__:main"]}, diff --git a/opentts-abc/install.sh b/opentts-abc/install.sh deleted file mode 100755 index 19284e4..0000000 --- a/opentts-abc/install.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Directory of *this* script -this_dir="$( cd "$( dirname "$0" )" && pwd )" - -# Path to virtual environment -: "${venv:=${this_dir}/.venv}" - -# Python binary to use -: "${PYTHON=python3}" - -# pip install command -: "${PIP_INSTALL=install}" - -python_version="$(${PYTHON} --version)" - -# Create virtual environment -echo "Creating virtual environment at ${venv} (${python_version})" -rm -rf "${venv}" -"${PYTHON}" -m venv "${venv}" -source "${venv}/bin/activate" - -# Install Python dependencies -echo 'Installing Python dependencies' -pip3 ${PIP_INSTALL} --upgrade pip -pip3 ${PIP_INSTALL} --upgrade wheel setuptools - -find "${this_dir}" -name 'requirements*.txt' -type f -print0 | \ - xargs -0 -n1 pip3 ${PIP_INSTALL} -r - -# ----------------------------------------------------------------------------- - -echo "OK" diff --git a/opentts-abc/requirements_dev.txt b/opentts-abc/requirements_dev.txt deleted file mode 100644 index 5318af0..0000000 --- a/opentts-abc/requirements_dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -black==22.1.0 -coverage==5.0.4 -flake8==3.7.9 -mypy==0.910 -pylint==2.10.2 -pytest==5.4.1 -pytest-cov==2.8.1 diff --git a/pyinstaller/mimic3.py b/pyinstaller/mimic3.py new file mode 100644 index 0000000..26dfff8 --- /dev/null +++ b/pyinstaller/mimic3.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# Copyright 2022 Mycroft AI Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +"""Stub for PyInstaller""" + +from mimic3_tts.__main__ import main + +main() diff --git a/pyinstaller/mimic3_server.py b/pyinstaller/mimic3_server.py new file mode 100644 index 0000000..bc3b8f5 --- /dev/null +++ b/pyinstaller/mimic3_server.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# Copyright 2022 Mycroft AI Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +"""Stub for PyInstaller""" + +from mimic3_http.__main__ import main + +main() diff --git a/mimic3-http/requirements_dev.txt b/requirements_dev.txt similarity index 86% rename from mimic3-http/requirements_dev.txt rename to requirements_dev.txt index 5318af0..3401b9b 100644 --- a/mimic3-http/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ -black==22.1.0 +black==22.3.0 coverage==5.0.4 flake8==3.7.9 mypy==0.910