diff --git a/mimic3-tts/mimic3_tts/tts.py b/mimic3-tts/mimic3_tts/tts.py
index f770743..955b57e 100644
--- a/mimic3-tts/mimic3_tts/tts.py
+++ b/mimic3-tts/mimic3_tts/tts.py
@@ -343,8 +343,19 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
         for sent_phonemes, break_type in voice.text_to_phonemes(
             text, text_language=text_language
         ):
+            add_major_silence = (break_type == BreakType.MAJOR) and (
+                major_break_ms is not None
+            )
+            add_minor_silence = (break_type == BreakType.MINOR) and (
+                minor_break_ms is not None
+            )
+
             # Utterances have start/end meta phonemes (usually ^ and $)
-            is_utterance = break_type != BreakType.NONE
+            is_utterance = (
+                (break_type == BreakType.UTTERANCE)
+                or add_major_silence
+                or add_minor_silence
+            )
 
             self._results.append(
                 Mimic3Phonemes(
@@ -355,9 +366,11 @@ class Mimic3TextToSpeechSystem(TextToSpeechSystem):
             )
 
             # Add silence if using manual break intervals
-            if (break_type == BreakType.MAJOR) and (major_break_ms is not None):
+            if add_major_silence:
+                assert major_break_ms is not None
                 self.add_break(major_break_ms)
-            elif (break_type == BreakType.MINOR) and (minor_break_ms is not None):
+            elif add_minor_silence:
+                assert minor_break_ms is not None
                 self.add_break(minor_break_ms)
 
     # pylint: disable=arguments-differ
diff --git a/mimic3-tts/mimic3_tts/voice.py b/mimic3-tts/mimic3_tts/voice.py
index fd790e5..7e8588c 100644
--- a/mimic3-tts/mimic3_tts/voice.py
+++ b/mimic3-tts/mimic3_tts/voice.py
@@ -401,7 +401,7 @@ class GruutVoice(Mimic3Voice):
         for sentence in gruut.sentences(text, lang=text_language):
             sent_phonemes = [w.phonemes for w in sentence if w.phonemes]
             if sent_phonemes:
-                yield sent_phonemes, BreakType.NONE
+                yield sent_phonemes, BreakType.UTTERANCE
 
     def word_to_phonemes(
         self,
@@ -589,7 +589,7 @@ class SymbolsVoice(Mimic3Voice):
         word_phonemes = [
             list(IPA.graphemes(wp_str)) for wp_str in text.split(word_separator)
         ]
-        yield word_phonemes, BreakType.NONE
+        yield word_phonemes, BreakType.UTTERANCE
 
 
 # -----------------------------------------------------------------------------