diff --git a/Dockerfile b/Dockerfile
index 5856586..5715b7d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -101,7 +101,7 @@ RUN export expected_sample="tests/apope_sample_${TARGETARCH}${TARGETVARIANT}.wav
     --voice 'en_UK/apope_low' \
     < tests/apope_sample.txt \
     > tests/actual_sample.wav && \
-    tests/samples_match.sh tests/actual_sample.wav "${expected_sample}"
+    tests/samples_match.py tests/actual_sample.wav "${expected_sample}"
 
 USER mimic3
 
diff --git a/Dockerfile.debian b/Dockerfile.debian
index e57c16f..1af4615 100644
--- a/Dockerfile.debian
+++ b/Dockerfile.debian
@@ -175,7 +175,7 @@ RUN export expected_sample="tests/apope_sample_${TARGETARCH}${TARGETVARIANT}.wav
     --voice 'en_UK/apope_low' \
     < tests/apope_sample.txt \
     > tests/actual_sample.wav && \
-    tests/samples_match.sh tests/actual_sample.wav "${expected_sample}"
+    tests/samples_match.py tests/actual_sample.wav "${expected_sample}"
 
 # -----------------------------------------------------------------------------
 
diff --git a/Dockerfile.debian.dockerignore b/Dockerfile.debian.dockerignore
index 8d58c97..13711e2 100644
--- a/Dockerfile.debian.dockerignore
+++ b/Dockerfile.debian.dockerignore
@@ -5,7 +5,7 @@
 !pyinstaller/mimic3-download
 !tests/apope_sample.txt
 !tests/apope_sample_*.wav
-!tests/samples_match.sh
+!tests/samples_match.py
 !voices/
 !wheels/
 
diff --git a/Dockerfile.dist b/Dockerfile.dist
index e48b71d..3026259 100644
--- a/Dockerfile.dist
+++ b/Dockerfile.dist
@@ -100,7 +100,7 @@ RUN export expected_sample="tests/apope_sample_${TARGETARCH}${TARGETVARIANT}.wav
     --voice 'en_UK/apope_low' \
     < tests/apope_sample.txt \
     > tests/actual_sample.wav && \
-    tests/samples_match.sh tests/actual_sample.wav "${expected_sample}"
+    tests/samples_match.py tests/actual_sample.wav "${expected_sample}"
 
 # -----------------------------------------------------------------------------
 
diff --git a/Dockerfile.dist.dockerignore b/Dockerfile.dist.dockerignore
index 42305cc..abd58fb 100644
--- a/Dockerfile.dist.dockerignore
+++ b/Dockerfile.dist.dockerignore
@@ -2,7 +2,7 @@
 !build-dist.sh
 !tests/apope_sample.txt
 !tests/apope_sample_*.wav
-!tests/samples_match.sh
+!tests/samples_match.py
 !voices/
 !wheels/
 !debian/control.in.*
diff --git a/Dockerfile.dockerignore b/Dockerfile.dockerignore
index 1924534..9fbc733 100644
--- a/Dockerfile.dockerignore
+++ b/Dockerfile.dockerignore
@@ -2,7 +2,7 @@
 !install.sh
 !tests/apope_sample.txt
 !tests/apope_sample_*.wav
-!tests/samples_match.sh
+!tests/samples_match.py
 !voices/
 !wheels/
 !debian/control.in.*
diff --git a/tests/samples_match.py b/tests/samples_match.py
new file mode 100755
index 0000000..a969677
--- /dev/null
+++ b/tests/samples_match.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright 2022 Mycroft AI Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# -----------------------------------------------------------------------------
+#
+# Compares two WAV files, exiting abnormally if they differ by a percentage.
+#
+import argparse
+import wave
+
+parser = argparse.ArgumentParser()
+parser.add_argument("wav1", help="First sample")
+parser.add_argument("wav2", help="Second sample")
+parser.add_argument(
+    "--percent-threshold",
+    type=float,
+    default=0.1,
+    help="Percent of samples allowed to be different",
+)
+args = parser.parse_args()
+
+with wave.open(args.wav1, "rb") as wav1, wave.open(args.wav2, "rb") as wav2:
+    assert wav1.getframerate() == wav2.getframerate(), "Mismatched sample rates"
+    assert wav1.getsampwidth() == wav2.getsampwidth(), "Mismatched sample widths"
+    assert wav1.getnchannels() == wav2.getnchannels(), "Mismatched channels"
+
+    wav1_samples = wav1.getnframes()
+    wav2_samples = wav2.getnframes()
+
+    smaller_samples = min(wav1_samples, wav2_samples)
+    assert smaller_samples > 0, "Empty WAV"
+
+    max_different = int(args.percent_threshold * smaller_samples)
+
+    # Mismatched size is starting difference
+    num_samples_different = abs(wav1_samples - wav2_samples)
+
+    for _ in range(smaller_samples):
+        # Check every sample
+        if num_samples_different > max_different:
+            break
+
+        if wav1.readframes(1) != wav2.readframes(1):
+            num_samples_different += 1
+
+    assert num_samples_different <= max_different, "Different"
+
+    percent_different = num_samples_different / smaller_samples
+    print(percent_different)
diff --git a/tests/samples_match.sh b/tests/samples_match.sh
deleted file mode 100755
index 72d38fe..0000000
--- a/tests/samples_match.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Copyright 2022 Mycroft AI Inc.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# -----------------------------------------------------------------------------
-#
-# Uses cmp to compare byte differences between two samples.
-# Exits with an error if differences exceed a threshold.
-#
-set -o pipefail
-
-if [ -z "$2" ]; then
-    echo 'Usage: samples_match.sh WAV1 WAV2 [threshold]';
-    exit 1
-fi
-
-wav1="$1"
-wav2="$2"
-threshold="${3:-5000}"
-
-bytes_different="$(cmp -l "${wav1}" "${wav2}" | wc -l)"
-
-if (( ${bytes_different} > ${threshold} )); then
-    echo "Samples differ too much (${bytes_different} > ${threshold})";
-    exit 1;
-fi
-
-echo 'OK'