From 75785e4ab067288e58ba0c0335272ac2de563521 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 15 Feb 2014 16:37:08 -0500 Subject: [PATCH 01/15] Refactored __str__ for superboxes. #175 --- glymur/jp2box.py | 83 ++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 63 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index a599f84..e358eab 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -88,6 +88,18 @@ class Jp2kBox(object): msg = "Not supported for {0} box.".format(self.longname) raise NotImplementedError(msg) + def _str_superbox(self): + """__str__ method for all superboxes.""" + msg = Jp2kBox.__str__(self) + for box in self.box: + boxstr = str(box) + + # Add indentation. + strs = [('\n ' + x) for x in boxstr.split('\n')] + msg += ''.join(strs) + return msg + + def _write_superbox(self, fptr): """Write a superbox. @@ -592,16 +604,7 @@ class CodestreamHeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -659,16 +662,7 @@ class CompositingLayerHeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -831,10 +825,10 @@ class ContiguousCodestreamBox(Jp2kBox): msg += '\n Main header:' for segment in self.main_header.segment: segstr = str(segment) - # Add indentation. strs = [('\n ' + x) for x in segstr.split('\n')] msg += ''.join(strs) + return msg @staticmethod @@ -1156,16 +1150,7 @@ class FragmentTableBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -1415,16 +1400,7 @@ class AssociationBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -1485,13 +1461,7 @@ class JP2HeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg def write(self, fptr): @@ -2086,13 +2056,7 @@ class ResolutionBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -2612,14 +2576,7 @@ class UUIDInfoBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - box_str = str(box) - - # Add indentation. - lst = [('\n ' + x) for x in box_str.split('\n')] - msg += ''.join(lst) - + msg = self._str_superbox() return msg @staticmethod From 3c333e999da41cb6129cd9ce346e5b72caf2c7e1 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 20:05:39 -0500 Subject: [PATCH 02/15] Parsing unknown superboxes with known 1st sub box. #175 --- glymur/jp2box.py | 22 +++++++++++++++++++++- glymur/test/test_jp2box_jpx.py | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index e358eab..4810834 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -164,12 +164,32 @@ class Jp2kBox(object): # Call the proper parser for the given box with ID "T". try: box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError: + except KeyError as err: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) box = Jp2kBox(box_id, offset=start, length=num_bytes, longname='Unknown box') + if fptr.tell() != start + 8: + # If the file pointer has advanced, then the KeyError + # ocurred during the parsing of the box. + pass + else: + # Could it be a superbox with recognizable child boxes? + # Peek ahead to see. + pos = fptr.tell() + read_buffer = fptr.read(8) + sub_length, sub_id = struct.unpack('>I4s', read_buffer) + sub_id = sub_id.decode('utf-8') + + # Regardless of whether or not we recognize the box, rewind back + # to properly advance to the next box. + fptr.seek(pos) + + # Now process any child boxes if we actually did recognize it. + if sub_id in _BOX_WITH_ID.keys(): + box.box = box.parse_superbox(fptr) + superbox.append(box) # Position to the start of the next box. diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 8b5e075..6624291 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -4,6 +4,7 @@ Test suite specifically targeting JPX box layout. """ import os +import shutil import struct import sys import tempfile @@ -202,6 +203,25 @@ class TestJPX(unittest.TestCase): self.assertEqual(jpx.box[2].standard_flag, (5, 42, 45, 2, 18, 19, 1, 8, 12, 31, 20)) + @unittest.skipIf(sys.hexversion < 0x03000000, "Needs unittest in 3.x.") + def test_unknown_superbox(self): + """Verify that we can handle an unknown superbox.""" + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with open(self.jpxfile, 'rb') as ifile: + tfile.write(ifile.read()) + + # Add the header for an unknwon superbox. + write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) + tfile.write(write_buffer) + write_buffer = struct.pack('>I4sI', 12, 'free'.encode(), 0) + tfile.write(write_buffer) + tfile.flush() + + with self.assertWarns(UserWarning): + jpx = Jp2k(tfile.name) + self.assertEqual(jpx.box[-1].box_id, 'grp ') + self.assertEqual(jpx.box[-1].box[0].box_id, 'free') + def test_free_box(self): """Verify that we can handle a free box.""" j = Jp2k(self.jpxfile) From 9999a60656592d0b6b613f5010a0f2863d30591a Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 21:43:05 -0500 Subject: [PATCH 03/15] Improved the warning when an exif tag is bad. #175 --- glymur/_uuid_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glymur/_uuid_io.py b/glymur/_uuid_io.py index d4971a1..ed610e5 100644 --- a/glymur/_uuid_io.py +++ b/glymur/_uuid_io.py @@ -146,7 +146,7 @@ class _Ifd(object): tag_name = tagnum2name[tag] except KeyError: # Ok, we don't recognize this tag. Just use the numeric id. - msg = 'Unrecognized Exif tag "{0}".'.format(tag) + msg = 'Unrecognized Exif tag: {0}'.format(tag) warnings.warn(msg, UserWarning) tag_name = tag self.processed_ifd[tag_name] = value From 331f901dae997b6babf9440d48fda26d33daba43 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 22:03:33 -0500 Subject: [PATCH 04/15] Refactored superbox parsing. #175 --- glymur/jp2box.py | 77 ++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 4810834..d2e374b 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -120,6 +120,53 @@ class Jp2kBox(object): fptr.write(struct.pack('>I', end_pos - orig_pos)) fptr.seek(end_pos) + def parse_this_box(self, fptr, box_id, start, num_bytes): + """Parse the current box. + + Parameters + ---------- + fptr : file + Open file object. + box_id : str + 4-letter identifier for the current box. + start, num_bytes: int + Byte offset and length of the current box. + + Returns + ------- + box : Jp2kBox + object corresponding to the current box + """ + try: + box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) + except KeyError as err: + msg = 'Unrecognized box ({0}) encountered.'.format(box_id) + warnings.warn(msg) + box = Jp2kBox(box_id, offset=start, length=num_bytes, + longname='Unknown box') + + if fptr.tell() != start + 8: + # If the file pointer has advanced, then the KeyError + # ocurred during the parsing of the box. + pass + else: + # Could it be a superbox with recognizable child boxes? + # Peek ahead to see. + pos = fptr.tell() + read_buffer = fptr.read(8) + sub_length, sub_id = struct.unpack('>I4s', read_buffer) + sub_id = sub_id.decode('utf-8') + + # Regardless of whether or not we recognize the box, rewind back + # to properly advance to the next box. + fptr.seek(pos) + + # Now process any child boxes if we actually did recognize it. + if sub_id in _BOX_WITH_ID.keys(): + box.box = box.parse_superbox(fptr) + + return box + def parse_superbox(self, fptr): """Parse a superbox (box consisting of nothing but other boxes. @@ -159,36 +206,10 @@ class Jp2kBox(object): num_bytes, = struct.unpack('>Q', read_buffer) else: + # The box_length value really is the length of the box! num_bytes = box_length - # Call the proper parser for the given box with ID "T". - try: - box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError as err: - msg = 'Unrecognized box ({0}) encountered.'.format(box_id) - warnings.warn(msg) - box = Jp2kBox(box_id, offset=start, length=num_bytes, - longname='Unknown box') - - if fptr.tell() != start + 8: - # If the file pointer has advanced, then the KeyError - # ocurred during the parsing of the box. - pass - else: - # Could it be a superbox with recognizable child boxes? - # Peek ahead to see. - pos = fptr.tell() - read_buffer = fptr.read(8) - sub_length, sub_id = struct.unpack('>I4s', read_buffer) - sub_id = sub_id.decode('utf-8') - - # Regardless of whether or not we recognize the box, rewind back - # to properly advance to the next box. - fptr.seek(pos) - - # Now process any child boxes if we actually did recognize it. - if sub_id in _BOX_WITH_ID.keys(): - box.box = box.parse_superbox(fptr) + box = self.parse_this_box(fptr, box_id, start, num_bytes) superbox.append(box) From df89b5523aafd43cb0aead80c66ddb719c97da01 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 22:16:46 -0500 Subject: [PATCH 05/15] Some pylint work. --- glymur/jp2box.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index d2e374b..ab8a6e5 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -63,6 +63,8 @@ class Jp2kBox(object): offset of the box from the start of the file. longname : str more verbose description of the box. + box : list + List of JPEG 2000 boxes. """ def __init__(self, box_id='', offset=0, length=0, longname=''): @@ -70,6 +72,7 @@ class Jp2kBox(object): self.length = length self.offset = offset self.longname = longname + self.box = [] def __repr__(self): msg = "glymur.jp2box.Jp2kBox(box_id='{0}', offset={1}, length={2}, " @@ -130,7 +133,7 @@ class Jp2kBox(object): box_id : str 4-letter identifier for the current box. start, num_bytes: int - Byte offset and length of the current box. + Byte offset and length of the current box. Returns ------- @@ -139,7 +142,7 @@ class Jp2kBox(object): """ try: box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError as err: + except KeyError: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) box = Jp2kBox(box_id, offset=start, length=num_bytes, @@ -154,7 +157,7 @@ class Jp2kBox(object): # Peek ahead to see. pos = fptr.tell() read_buffer = fptr.read(8) - sub_length, sub_id = struct.unpack('>I4s', read_buffer) + _, sub_id = struct.unpack('>I4s', read_buffer) sub_id = sub_id.decode('utf-8') # Regardless of whether or not we recognize the box, rewind back @@ -869,7 +872,7 @@ class ContiguousCodestreamBox(Jp2kBox): # Add indentation. strs = [('\n ' + x) for x in segstr.split('\n')] msg += ''.join(strs) - + return msg @staticmethod @@ -1725,8 +1728,7 @@ class PaletteBox(Jp2kBox): palette[j] = struct.unpack_from(fmt, read_buffer, offset=j * row_nbytes) - box = PaletteBox(palette, bps, signed, length=length, offset=offset) - return box + return PaletteBox(palette, bps, signed, length=length, offset=offset) # Map rreq codes to display text. From 5e76d1cf3927941b7563c3a52a006761447d6947 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 17 Feb 2014 17:38:26 -0500 Subject: [PATCH 06/15] Added print support for unknown boxes. #175 --- glymur/jp2box.py | 35 ++++++++++++++++++++++++-- glymur/test/fixtures.py | 8 ++++++ glymur/test/test_printing.py | 48 ++++++++++++++++++++---------------- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index ab8a6e5..043a1be 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -145,8 +145,8 @@ class Jp2kBox(object): except KeyError: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) - box = Jp2kBox(box_id, offset=start, length=num_bytes, - longname='Unknown box') + box = UnknownBox(box_id, offset=start, length=num_bytes, + longname='Unknown') if fptr.tell() != start + 8: # If the file pointer has advanced, then the KeyError @@ -2739,6 +2739,37 @@ class DataEntryURLBox(Jp2kBox): return box +class UnknownBox(Jp2kBox): + """Container for unrecognized boxes. + + Attributes + ---------- + box_id : str + 4-character identifier for the box. + length : int + length of the box in bytes. + offset : int + offset of the box from the start of the file. + longname : str + more verbose description of the box. + """ + def __init__(self, box_id, length=0, offset=-1, longname=''): + Jp2kBox.__init__(self, box_id=box_id, longname=longname) + self.length = length + self.offset = offset + + def __repr__(self): + msg = "glymur.jp2box.UnknownBox({0})".format(self.box_id) + return msg + + def __str__(self): + if len(self.box) > 0: + msg = self._str_superbox() + else: + msg = Jp2kBox.__str__(self) + return msg + + class UUIDBox(Jp2kBox): """Container for UUID box information. diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 6e0a76c..a7906c5 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -690,3 +690,11 @@ file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) Feature 060: 0x60 e-sRGB enumerated colorspace Feature 043: 0x40 Deprecated - compositing layer uses restricted ICC profile Vendor Features:""" + +file1_xml = r"""XML Box (xml ) @ (36, 439) + + + 2001-11-01T13:45:00.000-06:00 + Professional 120 Image + + """ diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 5fe3322..132d1ec 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -39,6 +39,7 @@ from .fixtures import text_gbr_27, text_gbr_33, text_gbr_34 class TestPrinting(unittest.TestCase): """Tests for verifying how printing works.""" def setUp(self): + self.jpxfile = glymur.data.jpxfile() self.jp2file = glymur.data.nemo() self.j2kfile = glymur.data.goodstuff() @@ -48,6 +49,31 @@ class TestPrinting(unittest.TestCase): def tearDown(self): pass + @unittest.skipIf(sys.hexversion < 0x03000000, "Needs unittest in 3.x.") + def test_unknown_superbox(self): + """Verify that we can handle an unknown superbox.""" + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with open(self.jpxfile, 'rb') as ifile: + tfile.write(ifile.read()) + + # Add the header for an unknwon superbox. + write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) + tfile.write(write_buffer) + write_buffer = struct.pack('>I4sI', 12, 'free'.encode(), 0) + tfile.write(write_buffer) + tfile.flush() + + with self.assertWarns(UserWarning): + jpx = Jp2k(tfile.name) + glymur.set_printoptions(short=True) + with patch('sys.stdout', new=StringIO()) as fake_out: + print(jpx.box[-1]) + actual = fake_out.getvalue().strip() + lines = ['Unknown Box (grp ) @ (695609, 20)', + ' Free Box (free) @ (695617, 12)'] + expected = '\n'.join(lines) + self.assertEqual(actual, expected) + def test_printoptions_bad_argument(self): """Verify error when bad parameter to set_printoptions""" with self.assertRaises(TypeError): @@ -602,27 +628,7 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(j.box[2]) actual = fake_out.getvalue().strip() - - lines = ['XML Box (xml ) @ (36, 439)', - ' ', - - ' ', - ' ' - + '2001-11-01T13:45:00.000-06:00' - + '', - - ' ' - + 'Professional 120 Image' - + '', - - ' ', - ' '] - expected = '\n'.join(lines) - self.assertEqual(actual, expected) + self.assertEqual(actual, fixtures.file1_xml) @unittest.skipIf(sys.hexversion < 0x03000000, "Only trusting python3 for printing non-ascii chars") From cd606e1f9df52dfe8207dac6fb15b7a3f5c7ae80 Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 18 Feb 2014 20:01:28 -0500 Subject: [PATCH 07/15] Starting write support for ftbl and flst boxes. #175 --- glymur/jp2box.py | 50 +++++++++++++++++++++-- glymur/test/test_jp2box_jpx.py | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 043a1be..4ce47ff 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -148,7 +148,8 @@ class Jp2kBox(object): box = UnknownBox(box_id, offset=start, length=num_bytes, longname='Unknown') - if fptr.tell() != start + 8: + cpos = fptr.tell() + if not ((cpos == start + 8) or (cpos == start + 16)): # If the file pointer has advanced, then the KeyError # ocurred during the parsing of the box. pass @@ -1122,6 +1123,18 @@ class FragmentListBox(Jp2kBox): self.length = length self.offset = offset + def _validate(self): + """Validate internal correctness.""" + if (((len(self.fragment_offset) != len(self.fragment_length)) or + (len(self.fragment_length) != len(self.data_reference)))): + msg = "The lengths of the fragment offsets, fragment lengths, and " + msg += "data reference items must be the same." + raise IOError(msg) + if any([x <= 0 for x in self.fragment_offset]): + raise IOError("Fragment offsets must all be positive.") + if any([x <= 0 for x in self.fragment_length]): + raise IOError("Fragment lengths must all be positive.") + def __repr__(self): msg = "glymur.jp2box.FragmentListBox()" return msg @@ -1141,6 +1154,22 @@ class FragmentListBox(Jp2kBox): return msg + def write(self, fptr): + """Write a fragment list box to file. + """ + self._validate() + num_items = len(self.fragment_offset) + length = 8 + 2 + num_items * 14 + fptr.write(struct.pack('>I', length)) + fptr.write(self.box_id.encode()) + fptr.write(struct.pack('>H', num_items)) + for j in range(num_items): + write_buffer = struct.pack('>QIH', + self.fragment_offset[j], + self.fragment_length[j], + self.data_reference[j]) + fptr.write(write_buffer) + @staticmethod def parse(fptr, offset, length): """Parse JPX free box. @@ -1184,10 +1213,11 @@ class FragmentTableBox(Jp2kBox): longname : str more verbose description of the box. """ - def __init__(self, length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='ftbl', longname='Fragment Table') self.length = length self.offset = offset + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.FragmentTableBox()" @@ -1212,7 +1242,7 @@ class FragmentTableBox(Jp2kBox): Returns ------- - FreeBox instance + FragmentTableBox instance """ box = FragmentTableBox(length=length, offset=offset) @@ -1222,6 +1252,20 @@ class FragmentTableBox(Jp2kBox): return box + def _validate(self): + """Self-validate the box before writing.""" + box_ids = [box.box_id for box in self.box] + if len(box_ids) != 1 or box_ids[0] != 'flst': + msg = "Fragment table boxes must have a single fragment list " + msg += "box as a child box." + raise IOError(msg) + + def write(self, fptr): + """Write a fragment table box to file. + """ + self._validate() + self._write_superbox(fptr) + class FreeBox(Jp2kBox): diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 6624291..99250c5 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -41,6 +41,34 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_ftbl(self): + """Write a fragment table box.""" + # Add a negative test where offset < 0 + # Add a negative test where length < 0 + # Add a negative test where ref > 0 but no data reference box. + # Add a negative test where more than one flst + # Add negative test where ftbl contained in a superbox. + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + # The ftyp box must be modified to jpx. + boxes[1].brand = 'jpx ' + boxes[1].compatibility_list = ['jp2 ', 'jpxb'] + + offset = [89] + length = [1132288] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + ftbl = glymur.jp2box.FragmentTableBox(box=[flst]) + boxes.append(ftbl) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + jpx = jp2.wrap(tfile.name, boxes=boxes) + + self.assertEqual(jpx.box[1].compatibility_list, ['jp2 ', 'jpxb']) + self.assertEqual(jpx.box[-1].box_id, 'ftbl') + self.assertEqual(jpx.box[-1].box[0].box_id, 'flst') + def test_jpxb_compatibility(self): """Wrap JP2 to JPX, state jpxb compatibility""" jp2 = Jp2k(self.jp2file) @@ -194,6 +222,51 @@ class TestJPX(unittest.TestCase): def tearDown(self): pass + def test_flst_lens_not_the_same(self): + """A fragment list box items must be the same length.""" + offset = [89] + length = [1132288] + reference = [0, 0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_flst_offsets_not_positive(self): + """A fragment list box offsets must be positive.""" + offset = [0] + length = [1132288] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_flst_lengths_not_positive(self): + """A fragment list box lengths must be positive.""" + offset = [89] + length = [0] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_ftbl_boxes_empty(self): + """A fragment table box must have at least one child box.""" + ftbl = glymur.jp2box.FragmentTableBox() + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftbl.write(tfile) + + def test_ftbl_child_not_flst(self): + """A fragment table box can only contain a fragment list.""" + free = glymur.jp2box.FreeBox() + ftbl = glymur.jp2box.FragmentTableBox(box=[free]) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftbl.write(tfile) + def test_jpx_rreq_mask_length_3(self): """There are some JPX files with rreq mask length of 3.""" jpx = Jp2k(self.jpxfile) From c82d525dd2286d95811a078b3237cf58641563ab Mon Sep 17 00:00:00 2001 From: jevans Date: Wed, 19 Feb 2014 20:25:08 -0500 Subject: [PATCH 08/15] FileType box sanity checks. #175 --- glymur/jp2box.py | 13 +++++++++++++ glymur/test/test_jp2box.py | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 4ce47ff..0a370f9 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1046,9 +1046,22 @@ class FileTypeBox(Jp2kBox): return msg + def _validate(self): + """Validate the box before writing to file.""" + if self.brand not in ['jp2 ', 'jpx ']: + msg = "The file type brand must be either 'jp2 ' or 'jpx '." + raise IOError(msg) + valid_cls = ['jp2 ', 'jpx ', 'jpxb'] + for item in self.compatibility_list: + if item not in valid_cls: + msg = "The file type compatibility list item '{0}' is not " + msg += "valid: valid entries are {1}" + raise IOError(msg.format(item, valid_cls)) + def write(self, fptr): """Write a File Type box to file. """ + self._validate() length = 16 + 4*len(self.compatibility_list) fptr.write(struct.pack('>I', length)) fptr.write('ftyp'.encode()) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index feffec9..426bf66 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -340,6 +340,29 @@ class TestChannelDefinition(unittest.TestCase): association=association) +class TestFileTypeBox(unittest.TestCase): + """Test suite for ftyp box issues.""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_brand_unknown(self): + """A ftyp box brand must be 'jp2 ' or 'jpx '.""" + ftyp = glymur.jp2box.FileTypeBox(brand='jp3') + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftyp.write(tfile) + + def test_cl_entry_unknown(self): + """A ftyp box cl list can only contain 'jp2 ', 'jpx ', or 'jpxb'.""" + ftyp = glymur.jp2box.FileTypeBox(compatibility_list=['jp3']) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftyp.write(tfile) + class TestColourSpecificationBox(unittest.TestCase): """Test suite for colr box instantiation.""" From 4276a978eab3e12516d9f077159b24b89048c002 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 20 Feb 2014 09:19:41 -0500 Subject: [PATCH 09/15] Allowing DataReferenceBoxes to be empty. #175 --- glymur/jp2box.py | 7 +++++-- glymur/test/test_jp2box_jpx.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 0a370f9..49cde5d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -915,9 +915,12 @@ class DataReferenceBox(Jp2kBox): DR : list Data Entry URL boxes. """ - def __init__(self, data_entry_url_boxes, length=0, offset=-1): + def __init__(self, data_entry_url_boxes=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='dtbl', longname='Data Reference') - self.DR = data_entry_url_boxes + if data_entry_url_boxes is None: + self.DR = [] + else: + self.DR = data_entry_url_boxes self.length = length self.offset = offset diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 99250c5..5d6cca2 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -127,6 +127,19 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box[2].box_id, 'lbl ') self.assertEqual(jpx.box[-1].box[2].label, label) + def test_empty_data_reference(self): + """Data reference boxes can be empty.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + dref = glymur.jp2box.DataReferenceBox() + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + jpx = jp2.wrap(tfile.name, boxes=boxes) + self.assertEqual(jpx.box[-1].box_id, 'dtbl') + self.assertEqual(len(jpx.box[-1].box), 0) + def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) From 4caa13a9559f4020f52fec56148803497acb9798 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 20 Feb 2014 22:34:52 -0500 Subject: [PATCH 10/15] More negative tests. #175 --- glymur/jp2box.py | 50 ++++++++++++++++++++++++++-------- glymur/test/test_jp2box.py | 12 +++++++- glymur/test/test_jp2box_jpx.py | 19 +++++++++++++ 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 49cde5d..f3fce20 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -29,6 +29,7 @@ import numpy as np from .codestream import Codestream from .core import _COLORSPACE_MAP_DISPLAY from .core import _COLOR_TYPE_MAP_DISPLAY +from .core import SRGB, GREYSCALE, YCC from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .core import ANY_ICC_PROFILE, VENDOR_COLOR_METHOD from .core import _pretty_print_xml @@ -270,13 +271,6 @@ class ColourSpecificationBox(Jp2kBox): approximation=0, colorspace=None, icc_profile=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='colr', longname='Colour Specification') - - if colorspace is not None and icc_profile is not None: - raise IOError("colorspace and icc_profile cannot both be set.") - if method not in (1, 2, 3, 4): - raise IOError("Invalid method.") - if approximation not in (0, 1, 2, 3, 4): - raise IOError("Invalid approximation.") self.method = method self.precedence = precedence self.approximation = approximation @@ -284,6 +278,33 @@ class ColourSpecificationBox(Jp2kBox): self.icc_profile = icc_profile self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + if self.colorspace is not None and self.icc_profile is not None: + raise IOError("colorspace and icc_profile cannot both be set.") + if self.method not in (1, 2, 3, 4): + raise IOError("Invalid method.") + if self.approximation not in (0, 1, 2, 3, 4): + raise IOError("Invalid approximation.") + + def _write_validate(self): + """In addition to constructor validation steps, run validation steps + for writing.""" + if self.colorspace is None: + msg = "Writing Colour Specification boxes without enumerated " + msg += "colorspaces is not supported at this time." + raise IOError(msg) + + if self.icc_profile is None: + if self.colorspace not in [SRGB, GREYSCALE, YCC]: + msg = "Colorspace should correspond to one of SRGB, GREYSCALE, " + msg += "or YCC." + raise IOError(msg) + + self._validate() + def __repr__(self): msg = "glymur.jp2box.ColourSpecificationBox(" @@ -327,10 +348,7 @@ class ColourSpecificationBox(Jp2kBox): def write(self, fptr): """Write an Colour Specification box to file. """ - if self.colorspace is None: - msg = "Writing Colour Specification boxes without enumerated " - msg += "colorspaces is not supported at this time." - raise NotImplementedError(msg) + self._write_validate() length = 15 if self.icc_profile is None else 11 + len(self.icc_profile) fptr.write(struct.pack('>I', length)) fptr.write('colr'.encode()) @@ -923,10 +941,20 @@ class DataReferenceBox(Jp2kBox): self.DR = data_entry_url_boxes self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + for box in self.DR: + if box.box_id != 'url ': + msg = 'All child boxes of a data reference box must be data ' + msg += 'entry URL boxes.' + raise IOError(msg) def write(self, fptr): """Write a Data Reference box to file. """ + self._validate() # Very similar to the say a superbox is written. orig_pos = fptr.tell() diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 426bf66..b6f0524 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -394,7 +394,7 @@ class TestColourSpecificationBox(unittest.TestCase): boxes = [self.jp2b, self.ftyp, self.jp2h, self.jp2c] boxes[2].box = [self.ihdr, ColourSpecificationBox(colorspace=None)] with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: - with self.assertRaises(NotImplementedError): + with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") @@ -440,6 +440,16 @@ class TestColourSpecificationBox(unittest.TestCase): glymur.jp2box.ColourSpecificationBox(colorspace=colorspace, approximation=approx) + def test_colr_with_bad_color(self): + """colr must have a valid color, strange as though that may sound.""" + colorspace = -1 + approx = 0 + colr = glymur.jp2box.ColourSpecificationBox(colorspace=colorspace, + approximation=approx) + with tempfile.TemporaryFile() as tfile: + with self.assertRaises(IOError): + colr.write(tfile) + class TestAppend(unittest.TestCase): """Tests for append method.""" diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 5d6cca2..b2916c5 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -140,6 +140,25 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box_id, 'dtbl') self.assertEqual(len(jpx.box[-1].box), 0) + def test_deurl_child_of_dtbl(self): + """Data reference boxes can only contain data entry url boxes.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + ftyp = glymur.jp2box.FileTypeBox() + with self.assertRaises(IOError): + dref = glymur.jp2box.DataReferenceBox([ftyp]) + + # Try to get around it by appending the ftyp box after creation. + dref = glymur.jp2box.DataReferenceBox() + dref.DR.append(ftyp) + + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jpx = jp2.wrap(tfile.name, boxes=boxes) + def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) From f8ced317dbf3da38861e69ede3e6fe86f5d35604 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 22 Feb 2014 21:30:52 -0500 Subject: [PATCH 11/15] More negative tests. Incompatible change to ChannelDefinitionBox. #175 --- CHANGES.txt | 21 +++++++++-------- docs/source/changelog.rst | 1 + docs/source/how_do_i.rst | 2 +- glymur/jp2box.py | 47 +++++++++++++++++++++++--------------- glymur/test/test_jp2box.py | 32 +++++++++++++++++++++++++- 5 files changed, 73 insertions(+), 30 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1febdfb..24d9616 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,13 +1,14 @@ -Feb 09, 2014 - Removed support for Python 2.6. Added write support for JP2 - UUID, DataEntryURL, Palette and Component Mapping boxes, JPX - Association, NumberList and DataReference boxes. Added read - support for JPX free, number list, data reference, fragment - table, and fragment list boxes. Improved JPX Reader Requirements box - support. Added get_printoptions, set_printoptions functions. - Palette box now a 2D numpy array instead of a list of 1D arrays. - JP2 super box constructors now take optional box list argument. - Fixed bug where JPX files with more than one codestream but - advertising jp2 compatibility were not being read. +Feb 09, 2014 - Changed constructor for ChannelDefinition box. Removed support + for Python 2.6. Added write support for JP2 UUID, DataEntryURL, + Palette and Component Mapping boxes, JPX Association, NumberList + and DataReference boxes. Added read support for JPX free, + number list, data reference, fragment table, and fragment list + boxes. Improved JPX Reader Requirements box support. Added + get_printoptions, set_printoptions functions. Palette box now + a 2D numpy array instead of a list of 1D arrays. JP2 super box + constructors now take optional box list argument. Fixed bug + where JPX files with more than one codestream but advertising + jp2 compatibility were not being read. Jan 28, 2014 - v0.5.10 Fixed bad warning when reader requirements box mask length is unsupported. diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index f7e5921..ad4e40b 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -11,6 +11,7 @@ ChangeLog * added write support for JP2 UUID, dataEntryURL, palette, and component mapping boxes * added read/write support for JPX free, number list, and data reference boxes * Added read support for JPX fragment list and fragment table boxes + * incompatible change to channel definition box constructor, channel_type and association are no longer keyword arguments * incompatible change to palette box constructor, it now takes a 2D numpy array instead of a list of 1D arrays 0.5.0 (September 16, 2013) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index a1fe089..fa8f098 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -359,7 +359,7 @@ channel, but we aren't doing that). :: >>> from glymur.core import RED, GREEN, BLUE, WHOLE_IMAGE >>> asoc = [RED, GREEN, BLUE, WHOLE_IMAGE] - >>> cdef = glymur.jp2box.ChannelDefinitionBox(channel_type=ctype, association=asoc) + >>> cdef = glymur.jp2box.ChannelDefinitionBox(ctype, asoc) >>> print(cdef) Channel Definition Box (cdef) @ (0, 0) Channel 0 (color) ==> (1) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index f3fce20..f82151a 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -544,23 +544,28 @@ class ChannelDefinitionBox(Jp2kBox): association : list index of the associated color """ - def __init__(self, index=None, channel_type=None, association=None, - **kwargs): + def __init__(self, channel_type, association, index=None, **kwargs): Jp2kBox.__init__(self, box_id='cdef', longname='Channel Definition') - # channel type and association must be specified. - if channel_type is None or association is None: - raise IOError("channel_type and association must be specified.") - if index is None: - index = list(range(len(channel_type))) + self.index = tuple(range(len(channel_type))) + else: + self.index = tuple(index) - if len(index) != len(channel_type) or len(index) != len(association): + self.channel_type = tuple(channel_type) + self.association = tuple(association) + self.__dict__.update(**kwargs) + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + # channel type and association must be specified. + if not (len(self.index) == len(self.channel_type) == len(self.association)): msg = "Length of channel definition box inputs must be the same." raise IOError(msg) # channel types must be one of 0, 1, 2, 65535 - if any(x not in [0, 1, 2, 65535] for x in channel_type): + if any(x not in [0, 1, 2, 65535] for x in self.channel_type): msg = "Channel types must be in the set of\n\n" msg += " 0 - colour image data for associated color\n" msg += " 1 - opacity\n" @@ -568,10 +573,6 @@ class ChannelDefinitionBox(Jp2kBox): msg += " 65535 - unspecified" raise IOError(msg) - self.index = tuple(index) - self.channel_type = tuple(channel_type) - self.association = tuple(association) - self.__dict__.update(**kwargs) def __str__(self): msg = Jp2kBox.__str__(self) @@ -597,6 +598,7 @@ class ChannelDefinitionBox(Jp2kBox): def write(self, fptr): """Write a channel definition box to file. """ + self._validate() num_components = len(self.association) fptr.write(struct.pack('>I', 8 + 2 + num_components * 6)) fptr.write('cdef'.encode('utf-8')) @@ -634,9 +636,10 @@ class ChannelDefinitionBox(Jp2kBox): channel_type = data[1:num_components * 6:3] association = data[2:num_components * 6:3] - box = ChannelDefinitionBox(index=index, channel_type=channel_type, - association=association, length=length, - offset=offset) + box = ChannelDefinitionBox(index=tuple(index), + channel_type=tuple(channel_type), + association=tuple(association), + length=length, offset=offset) return box @@ -795,7 +798,6 @@ class ComponentMappingBox(Jp2kBox): if _printoptions['short'] == True: return msg - for k in range(len(self.component_index)): if self.mapping_type[k] == 1: msg += '\n Component {0} ==> palette column {1}' @@ -1050,7 +1052,6 @@ class FileTypeBox(Jp2kBox): self.brand = brand self.minor_version = minor_version if compatibility_list is None: - # see W0102, pylint self.compatibility_list = ['jp2 '] else: self.compatibility_list = compatibility_list @@ -1718,6 +1719,15 @@ class PaletteBox(Jp2kBox): self.signed = signed self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + if ((len(self.bits_per_component) != len(self.signed)) or + (len(self.signed) != self.palette.shape[1])): + msg = "The length of the 'bits_per_component' and the 'signed' " + msg += "members must equal the number of columns of the palette." + raise IOError(msg) def __repr__(self): msg = "glymur.jp2box.PaletteBox({0}, bits_per_component={1}, " @@ -1737,6 +1747,7 @@ class PaletteBox(Jp2kBox): def write(self, fptr): """Write a Palette box to file. """ + self._validate() bytes_per_row = sum(self.bits_per_component) / 8 bytes_per_palette = bytes_per_row * self.palette.shape[0] box_length = 8 + 3 + self.palette.shape[1] + bytes_per_palette diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index b6f0524..fb15e26 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -139,7 +139,7 @@ class TestChannelDefinition(unittest.TestCase): def test_cdef_no_inputs(self): """channel_type and association are required inputs.""" - with self.assertRaises(IOError): + with self.assertRaises(TypeError): glymur.jp2box.ChannelDefinitionBox() def test_rgb_with_index(self): @@ -451,6 +451,36 @@ class TestColourSpecificationBox(unittest.TestCase): colr.write(tfile) +@unittest.skipIf(os.name == "nt", + "Problems using NamedTemporaryFile on windows.") +class TestPaletteBox(unittest.TestCase): + """Test suite for pclr box instantiation.""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_mismatched_bitdepth_signed(self): + """bitdepth and signed arguments must have equal length""" + palette = np.array([[255, 0, 255], [0, 255, 0]], dtype=np.uint8) + bps = (8, 8, 8) + signed = (False, False) + with self.assertRaises(IOError): + pclr = glymur.jp2box.PaletteBox(palette, bits_per_component=bps, + signed=signed) + + def test_mismatched_signed_palette(self): + """bitdepth and signed arguments must have equal length""" + palette = np.array([[255, 0, 255], [0, 255, 0]], dtype=np.uint8) + bps = (8, 8, 8, 8) + signed = (False, False, False, False) + with self.assertRaises(IOError): + pclr = glymur.jp2box.PaletteBox(palette, bits_per_component=bps, + signed=signed) + + class TestAppend(unittest.TestCase): """Tests for append method.""" From 16ce2ef88380dcad85e2edfda957b46ec9a29948 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 23 Feb 2014 12:37:47 -0500 Subject: [PATCH 12/15] More negative tests, pylint work. #175 --- glymur/jp2box.py | 17 ++++++--- glymur/jp2k.py | 17 ++++++++- glymur/test/test_jp2box_jpx.py | 65 ++++++++++++++++++++++------------ 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index f82151a..18f994d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -124,7 +124,7 @@ class Jp2kBox(object): fptr.write(struct.pack('>I', end_pos - orig_pos)) fptr.seek(end_pos) - def parse_this_box(self, fptr, box_id, start, num_bytes): + def _parse_this_box(self, fptr, box_id, start, num_bytes): """Parse the current box. Parameters @@ -214,7 +214,7 @@ class Jp2kBox(object): # The box_length value really is the length of the box! num_bytes = box_length - box = self.parse_this_box(fptr, box_id, start, num_bytes) + box = self._parse_this_box(fptr, box_id, start, num_bytes) superbox.append(box) @@ -560,7 +560,8 @@ class ChannelDefinitionBox(Jp2kBox): def _validate(self): """Verify that the box obeys the specifications.""" # channel type and association must be specified. - if not (len(self.index) == len(self.channel_type) == len(self.association)): + if not ((len(self.index) == len(self.channel_type)) and + (len(self.channel_type) == len(self.association))): msg = "Length of channel definition box inputs must be the same." raise IOError(msg) @@ -953,10 +954,18 @@ class DataReferenceBox(Jp2kBox): msg += 'entry URL boxes.' raise IOError(msg) + def _write_validate(self): + """Verify that the box obeys the specifications for writing. + """ + if len(self.DR) == 0: + msg = "A data reference box cannot be empty when written to a file." + raise IOError(msg) + self._validate() + def write(self, fptr): """Write a Data Reference box to file. """ - self._validate() + self._write_validate() # Very similar to the say a superbox is written. orig_pos = fptr.tell() diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 96b6e68..988e814 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1110,6 +1110,10 @@ def _validate_nonzero_image_size(nrows, ncols, component_index): raise IOError(msg) +JP2_IDS = ['colr', 'cdef', 'cmap', 'jp2c', 'ftyp', 'ihdr', 'jp2h', 'jP ', + 'pclr', 'res ', 'resc', 'resd', 'xml ', 'ulst', 'uinf', 'url ', + 'uuid'] + def _validate_jp2_box_sequence(boxes): """Run through series of tests for JP2 box legality. @@ -1118,6 +1122,18 @@ def _validate_jp2_box_sequence(boxes): _validate_signature_compatibility(boxes) _validate_jp2h(boxes) _validate_jp2c(boxes) + if boxes[1].brand == 'jpx ': + _validate_jpx_box_sequence(boxes) + else: + count = _collect_box_count(boxes) + for id in count.keys(): + if id not in JP2_IDS: + msg = "The presence of a '{0}' box requires that the file type " + msg += "brand be set to 'jpx '." + raise IOError(msg.format(id)) + +def _validate_jpx_box_sequence(boxes): + """Run through series of tests for JPX box legality.""" _validate_association(boxes) _validate_label(boxes) _validate_jpx_brand(boxes, boxes[1].brand) @@ -1125,7 +1141,6 @@ def _validate_jp2_box_sequence(boxes): _validate_singletons(boxes) _validate_top_level(boxes) - def _validate_signature_compatibility(boxes): """Validate the file signature and compatibility status.""" # Check for a bad sequence of boxes. diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index b2916c5..e50d4f9 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -4,12 +4,10 @@ Test suite specifically targeting JPX box layout. """ import os -import shutil import struct import sys import tempfile import unittest -import warnings import xml.etree.cElementTree as ET import glymur @@ -41,6 +39,18 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_jp2_with_jpx_box(self): + """If the brand is jp2, then no jpx boxes are allowed.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + boxes = jp2.box + + boxes.append(glymur.jp2box.AssociationBox()) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jp2.wrap(tfile.name, boxes=boxes) + def test_ftbl(self): """Write a fragment table box.""" # Add a negative test where offset < 0 @@ -128,17 +138,18 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box[2].label, label) def test_empty_data_reference(self): - """Data reference boxes can be empty.""" + """Empty data reference boxes can be created, but not written.""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + boxes[1].brand = 'jpx ' + dref = glymur.jp2box.DataReferenceBox() boxes.append(dref) with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: - jpx = jp2.wrap(tfile.name, boxes=boxes) - self.assertEqual(jpx.box[-1].box_id, 'dtbl') - self.assertEqual(len(jpx.box[-1].box), 0) + with self.assertRaises(IOError): + jp2.wrap(tfile.name, boxes=boxes) def test_deurl_child_of_dtbl(self): """Data reference boxes can only contain data entry url boxes.""" @@ -157,13 +168,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + flag = 0 version = (0, 0, 0) url = 'file:////usr/local/bin' @@ -174,16 +188,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_lbl_at_top_level(self): """Label boxes can only be inside a asoc box .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] - flag = 0 - version = (0, 0, 0) - url = 'file:////usr/local/bin' + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + lblb = glymur.jp2box.LabelBox('hi there') # Put it inside the jp2 header box. @@ -191,13 +205,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) - def test_data_reference_not_at_top_level(self): + def test_data_reference_in_subbox(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + flag = 0 version = (0, 0, 0) url = 'file:////usr/local/bin' @@ -209,13 +226,17 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_jp2_to_jpx_sans_jp2_compatibility(self): """jp2 wrapped to jpx not including jp2 compatibility is wrong.""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' boxes[1].compatibility_list.append('jp2 ') + numbers = [0, 1] nlst = glymur.jp2box.NumberListBox(numbers) the_xml = ET.fromstring('0') @@ -225,7 +246,7 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(RuntimeError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_jp2_to_jpx_sans_jpx_brand(self): """Verify error when jp2 wrapped to jpx does not include jpx brand.""" @@ -241,7 +262,7 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(RuntimeError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") @@ -262,7 +283,7 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_flst_offsets_not_positive(self): """A fragment list box offsets must be positive.""" @@ -272,7 +293,7 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_flst_lengths_not_positive(self): """A fragment list box lengths must be positive.""" @@ -282,14 +303,14 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_ftbl_boxes_empty(self): """A fragment table box must have at least one child box.""" ftbl = glymur.jp2box.FragmentTableBox() with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - ftbl.write(tfile) + ftbl.write(tfile) def test_ftbl_child_not_flst(self): """A fragment table box can only contain a fragment list.""" @@ -297,7 +318,7 @@ class TestJPX(unittest.TestCase): ftbl = glymur.jp2box.FragmentTableBox(box=[free]) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - ftbl.write(tfile) + ftbl.write(tfile) def test_jpx_rreq_mask_length_3(self): """There are some JPX files with rreq mask length of 3.""" @@ -314,7 +335,7 @@ class TestJPX(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: with open(self.jpxfile, 'rb') as ifile: tfile.write(ifile.read()) - + # Add the header for an unknwon superbox. write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) tfile.write(write_buffer) From 992c036da701ab297bca4bada4d7e3507c9ed5a1 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 24 Feb 2014 21:12:24 -0500 Subject: [PATCH 13/15] Added ftbl/data reference test. Removed bad asoc check. #175 --- glymur/jp2k.py | 24 ++++++------------------ glymur/test/test_jp2box_jpx.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 988e814..6db0198 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1134,7 +1134,6 @@ def _validate_jp2_box_sequence(boxes): def _validate_jpx_box_sequence(boxes): """Run through series of tests for JPX box legality.""" - _validate_association(boxes) _validate_label(boxes) _validate_jpx_brand(boxes, boxes[1].brand) _validate_jpx_compatibility(boxes, boxes[1].compatibility_list) @@ -1283,6 +1282,12 @@ def _validate_top_level(boxes): if 'dtbl' in multiples: raise IOError('There can only be one dtbl box in a file.') + # If there is one data reference box, then there must also be one ftbl. + if 'dtbl' in count and 'ftbl' not in count: + msg = 'The presence of a data reference box requires the presence of ' + msg += 'a fragment table box as well.' + raise IOError(msg) + def _validate_singletons(boxes): """Several boxes can only occur once.""" count = _collect_box_count(boxes) @@ -1336,23 +1341,6 @@ def _validate_label(boxes): # Same set of checks on any child boxes. _validate_label(box.box) -def _validate_association(boxes): - """ - Association boxes can only contain number list boxes and xml boxes, as far - as we know. - """ - for box in boxes: - if box.box_id == 'asoc': - if box.box[0].box_id != 'nlst' or box.box[1].box_id != 'xml ': - msg = "An Association box can only contain a NumberList box " - msg += "followed by an XML box." - raise RuntimeError(msg) - if hasattr(box, 'box') != 0: - # Same set of checks on any child boxes. - _validate_association(box.box) - - - def extract_image_cube(image): """Extract 3D image from openjpeg data structure. """ diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index e50d4f9..1eb5198 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -270,6 +270,7 @@ class TestJPX(unittest.TestCase): """Test suite for other JPX boxes.""" def setUp(self): + self.jp2file = glymur.data.nemo() self.jpxfile = glymur.data.jpxfile() def tearDown(self): @@ -354,6 +355,25 @@ class TestJPX(unittest.TestCase): self.assertEqual(j.box[16].box[0].box_id, 'free') self.assertEqual(type(j.box[16].box[0]), glymur.jp2box.FreeBox) + def test_data_reference_requires_dtbl(self): + """The existance of a data reference box requires a ftbl box as well.""" + flag = 0 + version = (0, 0, 0) + url1 = 'file:////usr/local/bin' + url2 = 'http://glymur.readthedocs.org' + jpx1 = glymur.Jp2k(self.jp2file) + boxes = jpx1.box + boxes[1].brand = 'jpx ' + + deurl1 = glymur.jp2box.DataEntryURLBox(flag, version, url1) + deurl2 = glymur.jp2box.DataEntryURLBox(flag, version, url2) + dref = glymur.jp2box.DataReferenceBox([deurl1, deurl2]) + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with self.assertRaises(IOError): + jpx2 = jpx1.wrap(tfile.name, boxes=boxes) + def test_dtbl(self): """Verify that we can interpret Data Reference boxes.""" # Copy the existing JPX file, add a data reference box onto the end. From 99d1a584d8619f75c7218ba920f935886c44eb8a Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 25 Feb 2014 07:15:12 -0500 Subject: [PATCH 14/15] Verifying that data entry url box URLs are null terminated. #175 --- glymur/jp2box.py | 9 +++++++-- glymur/test/test_jp2box.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 18f994d..3cf637d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2788,13 +2788,18 @@ class DataEntryURLBox(Jp2kBox): def write(self, fptr): """Write a data entry url box to file. """ - length = 8 + 1 + 3 + len(self.url.encode()) + # Make sure it is written out as null-terminated. + url = self.url + if self.url[-1] != chr(0): + url = url + chr(0) + + length = 8 + 1 + 3 + len(url.encode()) write_buffer = struct.pack('>I4sBBBB', length, self.box_id.encode(), self.version, self.flag[0], self.flag[1], self.flag[2]) fptr.write(write_buffer) - fptr.write(self.url.encode()) + fptr.write(url.encode()) def __repr__(self): diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index fb15e26..5861971 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -77,6 +77,28 @@ class TestDataEntryURL(unittest.TestCase): self.assertEqual(jp22.box[4].flag, (0, 0, 0)) self.assertEqual(jp22.box[4].url, url) + def test_null_termination(self): + """I.9.3.2 specifies that the location field must be null terminated.""" + jp2 = Jp2k(self.jp2file) + + url = 'http://glymur.readthedocs.org' + deurl = glymur.jp2box.DataEntryURLBox(0, (0, 0, 0), url) + boxes = [box for box in jp2.box if box.box_id != 'uuid'] + boxes.append(deurl) + with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: + jp22 = jp2.wrap(tfile.name, boxes=boxes) + + self.assertEqual(jp22.box[-1].length, 42) + + # Go to the last box. Seek past the L, T, version, and flag fields. + with open(tfile.name, 'rb') as fptr: + fptr.seek(jp22.box[-1].offset + 4 + 4 + 1 + 3) + + nbytes = jp22.box[-1].offset + jp22.box[-1].length - fptr.tell() + read_buffer = fptr.read(nbytes) + read_url = read_buffer.decode('utf-8') + self.assertEqual(url + chr(0), read_url) + @unittest.skipIf(glymur.version.openjpeg_version_tuple[0] < 2 or OPENJP2_IS_V2_OFFICIAL, From 04b143e1ff59cae24b9ad390d321d3e253535e4a Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 25 Feb 2014 21:41:02 -0500 Subject: [PATCH 15/15] Added proof of concept test for linking exterior codestreams. Closes #175 --- glymur/test/test_jp2box_jpx.py | 60 ++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 1eb5198..7493dea 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -12,7 +12,8 @@ import xml.etree.cElementTree as ET import glymur from glymur import Jp2k - +from glymur.jp2box import DataEntryURLBox, FileTypeBox, JPEG2000SignatureBox +from glymur.jp2box import DataReferenceBox, FragmentListBox, FragmentTableBox @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") class TestJPXWrap(unittest.TestCase): @@ -20,6 +21,7 @@ class TestJPXWrap(unittest.TestCase): def setUp(self): self.jp2file = glymur.data.nemo() + self.j2kfile = glymur.data.goodstuff() raw_xml = b""" @@ -39,6 +41,60 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_jpx_ftbl_no_codestream(self): + """Can have a jpx with no codestream.""" + with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile1: + with open(self.jp2file, 'rb') as fptr: + tfile1.write(fptr.read()) + tfile1.flush() + jp2_1 = Jp2k(tfile1.name) + + jp2c = [box for box in jp2_1.box if box.box_id == 'jp2c'][0] + + # coff and clen will be the offset and length input arguments + # to the fragment list box. dr_idx is the data reference index. + coff = [] + clen = [] + dr_idx = [] + + coff.append(jp2c.offset + 8) + clen.append(jp2c.length - (coff[0] - jp2c.offset)) + dr_idx.append(1) + + # Make the url box for this codestream. + url1 = DataEntryURLBox(0, [0, 0, 0], 'file://' + tfile1.name) + + with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile2: + + j2k = Jp2k(self.j2kfile) + jp2_2 = j2k.wrap(tfile2.name) + + jp2c = [box for box in jp2_2.box if box.box_id == 'jp2c'][0] + coff.append(jp2c.offset + 8) + clen.append(jp2c.length - (coff[0] - jp2c.offset)) + dr_idx.append(2) + + # Make the url box for this codestream. + url2 = DataEntryURLBox(0, [0, 0, 0], 'file://' + tfile2.name) + + boxes = [JPEG2000SignatureBox(), + FileTypeBox(brand='jpx ', + compatibility_list=['jpx ', + 'jp2 ', 'jpxb']), + jp2_1.box[2]] + with tempfile.NamedTemporaryFile(suffix='.jpx') as tjpx: + for box in boxes: + box.write(tjpx) + + flst = FragmentListBox(coff, clen, dr_idx) + ftbl = FragmentTableBox([flst]) + ftbl.write(tjpx) + + boxes = [url1, url2] + dtbl = DataReferenceBox(data_entry_url_boxes=boxes) + dtbl.write(tjpx) + tjpx.flush() + def test_jp2_with_jpx_box(self): """If the brand is jp2, then no jpx boxes are allowed.""" jp2 = Jp2k(self.jp2file) @@ -372,7 +428,7 @@ class TestJPX(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: with self.assertRaises(IOError): - jpx2 = jpx1.wrap(tfile.name, boxes=boxes) + jpx1.wrap(tfile.name, boxes=boxes) def test_dtbl(self): """Verify that we can interpret Data Reference boxes."""