From 297921d6830f2fedfc69d05083d2a281c614c12e Mon Sep 17 00:00:00 2001 From: Michael Hansen Date: Thu, 5 Aug 2021 16:40:47 -0400 Subject: [PATCH] Add phoneme counts --- espeak_phonemizer/__init__.py | 2 +- espeak_phonemizer/phoneme_ids.py | 92 +++++++++++++++++++++++++++++++- setup.py | 1 - 3 files changed, 92 insertions(+), 3 deletions(-) diff --git a/espeak_phonemizer/__init__.py b/espeak_phonemizer/__init__.py index de6e071..85287ef 100644 --- a/espeak_phonemizer/__init__.py +++ b/espeak_phonemizer/__init__.py @@ -15,7 +15,7 @@ class Phonemizer: espeakCHARS_AUTO = 0 espeakPHONEMES = 0x100 - LANG_SWITCH_FLAG = re.compile(r"\([^)]+\)") + LANG_SWITCH_FLAG = re.compile(r"\([^)]*\)") DEFAULT_CLAUSE_BREAKERS = {",", ";", ":", ".", "!", "?"} diff --git a/espeak_phonemizer/phoneme_ids.py b/espeak_phonemizer/phoneme_ids.py index 477ec9f..f27b6f2 100644 --- a/espeak_phonemizer/phoneme_ids.py +++ b/espeak_phonemizer/phoneme_ids.py @@ -1,8 +1,11 @@ import argparse import csv +import itertools import logging import os import sys +import typing +from collections import Counter _LOGGER = logging.getLogger("phoneme_ids") @@ -62,10 +65,26 @@ def main(): action="store_true", help="Pull primary/secondary stress out as separate phonemes", ) + parser.add_argument( + "--write-phoneme-counts", help="Path to write phoneme counts observed in input" + ) + parser.add_argument( + "-m", + "--map", + nargs=2, + action="append", + help="Map from observed phoneme to desired phonemes", + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO) + # Map from observed phonemes to desired + phoneme_map = {} + if args.map: + for from_phoneme, to_phoneme in args.map: + phoneme_map[from_phoneme] = to_phoneme + phoneme_to_id = {} if args.read_phonemes: @@ -124,6 +143,7 @@ def main(): # Read all input and get set of phonemes all_phonemes = set(phoneme_to_id.keys()) + all_phoneme_counts = Counter() if args.simple_punctuation: # Add , and . @@ -163,7 +183,10 @@ def main(): if args.simple_punctuation: phoneme = _PUNCTUATION_MAP.get(phoneme, phoneme) + phoneme = phoneme_map.get(phoneme, phoneme) + all_phonemes.add(phoneme) + all_phoneme_counts[phoneme] += 1 # Assign phonemes to ids in sorted order for phoneme in sorted(all_phonemes): @@ -191,7 +214,7 @@ def main(): if args.separate_stress: # Split stress out while phoneme and (phoneme[0] in _STRESS): - stress = phoneme[0] + stress = phoneme_map.get(phoneme[0], phoneme[0]) word_ids.append(phoneme_to_id[stress]) phoneme = phoneme[1:] @@ -199,6 +222,7 @@ def main(): if args.simple_punctuation: phoneme = _PUNCTUATION_MAP.get(phoneme, phoneme) + phoneme = phoneme_map.get(phoneme, phoneme) word_ids.append(phoneme_to_id[phoneme]) if word_ids: @@ -237,6 +261,72 @@ def main(): ): print(phoneme_id, phoneme, file=phonemes_file) + if args.write_phoneme_counts: + # Write file with PHONEMECOUNT format + with open(args.write_phoneme_counts, "w") as phoneme_counts_file: + for phoneme, phoneme_count in all_phoneme_counts.most_common(): + print(phoneme, phoneme_count, file=phoneme_counts_file) + + +# ----------------------------------------------------------------------------- + + +def phonemes_to_ids( + word_phonemes: typing.List[typing.List[str]], + phoneme_to_id: typing.Mapping[str, int], + pad: typing.Optional[str] = None, + bos: typing.Optional[str] = None, + eos: typing.Optional[str] = None, + blank: typing.Optional[str] = None, + simple_punctuation: bool = False, + separate_stress: bool = False, + phoneme_map: typing.Optional[typing.Mapping[str, str]] = None, +) -> typing.List[int]: + if phoneme_map is None: + phoneme_map = {} + + blank_id: typing.Optional[int] = None + if blank: + blank_id = phoneme_to_id[blank] + + # Transform into phoneme ids + word_phoneme_ids = [] + + # Add beginning-of-sentence symbol + if bos: + word_phoneme_ids.append([phoneme_to_id[bos]]) + + if blank_id is not None: + word_phoneme_ids.append([blank_id]) + + for word in word_phonemes: + word_ids = [] + for phoneme in word: + if separate_stress: + # Split stress out + while phoneme and (phoneme[0] in _STRESS): + stress = phoneme_map.get(phoneme[0], phoneme[0]) + word_ids.append(phoneme_to_id[stress]) + phoneme = phoneme[1:] + + if phoneme: + if simple_punctuation: + phoneme = _PUNCTUATION_MAP.get(phoneme, phoneme) + + phoneme = phoneme_map.get(phoneme, phoneme) + word_ids.append(phoneme_to_id[phoneme]) + + if word_ids: + word_phoneme_ids.append(word_ids) + if blank_id is not None: + word_phoneme_ids.append([blank_id]) + + # Add end-of-sentence symbol + if eos: + word_phoneme_ids.append([phoneme_to_id[eos]]) + + return list(itertools.chain.from_iterable(word_phoneme_ids)) + # ----------------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 5301b73..0ef6122 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ setuptools.setup( url="https://github.com/synesthesiam/espeak-phonemizer", packages=setuptools.find_packages(), package_data={"espeak_phonemizer": ["VERSION", "py.typed"]}, - install_requires=requirements, entry_points={ "console_scripts": [ "espeak-phonemizer = espeak_phonemizer.__main__:main",