diff --git a/tests/unit/split_buffer_test.py b/tests/unit/split_buffer_test.py index a78e99a6..41dc50e4 100644 --- a/tests/unit/split_buffer_test.py +++ b/tests/unit/split_buffer_test.py @@ -6,32 +6,47 @@ from .. import unittest class SplitBufferTest(unittest.TestCase): def test_single_line_chunks(self): def reader(): - yield "abc\n" - yield "def\n" - yield "ghi\n" + yield b'abc\n' + yield b'def\n' + yield b'ghi\n' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi\n"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi\n']) def test_no_end_separator(self): def reader(): - yield "abc\n" - yield "def\n" - yield "ghi" + yield b'abc\n' + yield b'def\n' + yield b'ghi' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) def test_multiple_line_chunk(self): def reader(): - yield "abc\ndef\nghi" + yield b'abc\ndef\nghi' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "def\n", "ghi"]) + self.assert_produces(reader, [b'abc\n', b'def\n', b'ghi']) def test_chunked_line(self): def reader(): - yield "a" - yield "b" - yield "c" - yield "\n" - yield "d" + yield b'a' + yield b'b' + yield b'c' + yield b'\n' + yield b'd' - self.assertEqual(list(split_buffer(reader(), '\n')), ["abc\n", "d"]) + self.assert_produces(reader, [b'abc\n', b'd']) + + def test_preserves_unicode_sequences_within_lines(self): + string = u"a\u2022c\n".encode('utf-8') + + def reader(): + yield string + + self.assert_produces(reader, [string]) + + def assert_produces(self, reader, expectations): + split = split_buffer(reader(), b'\n') + + for (actual, expected) in zip(split, expectations): + self.assertEqual(type(actual), type(expected)) + self.assertEqual(actual, expected)