From b740322b3812d020f8b45123c89fcdde141ad1ab Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 21 Jul 2013 11:08:55 -0400 Subject: [PATCH 01/30] Minor doc bugs fixed, closes #79. --- README.md | 6 +++--- docs/source/detailed_installation.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b8d38c0..d8ea732 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ glymur: a Python interface for JPEG 2000 ========================================= -**glymur** contains a Python interface to the OpenJPEG library -which allows linux and mac users to read and write JPEG 2000 files. -**glymur** works on Python 2.6, 2.7 and 3.3. Python 3.3 is strongly +**glymur** contains a Python interface to the OpenJPEG library which +allows one to read and write JPEG 2000 files. **glymur** works on +Python 2.6, 2.7 and 3.3. Python 3.3 is strongly recommended. Please read the docs, https://glymur.readthedocs.org/en/latest/ diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index 872f373..06c6b79 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -115,7 +115,7 @@ following RPMs installed. * numpy * matplotlib (optional) -In addition, you must install contextlib2 and Pillow via pip. +In addition, you must install contextlib2 and Pillow via pip. :: $ yum install python-devel # pip needs this in order to compile Pillow $ pip-python install Pillow --user From 64faebd06ca089fffd1c0b4787d909141d4b0c1f Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 21 Jul 2013 11:25:46 -0400 Subject: [PATCH 02/30] Prepping for 0.2.4 bug fix release. --- CHANGES.txt | 4 ++++ docs/source/conf.py | 2 +- setup.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index b1ab624..f80b04b 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +Jul 21, 2013 - v0.2.4 Fixed markdown bug for Fedora 17 information, fixed + out-of-date windows information (issue79). Fixed incorrect + interpretation of Psot parameter (issue78). + Jul 18, 2013 - v0.2.3 Support for Python 2.6, OpenJPEG 1.4. Incompatible change to ChannelDefinitionBox constructor. Added RGBA example. diff --git a/docs/source/conf.py b/docs/source/conf.py index 96eeba9..bd59b1b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,7 +78,7 @@ copyright = u'2013, John Evans' # The short X.Y version. version = '0.1' # The full version, including alpha/beta/rc tags. -release = '0.2.3' +release = '0.2.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 1f17ddc..ba8d257 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import sys kwargs = {'name': 'Glymur', - 'version': '0.2.3', + 'version': '0.2.4rc1', 'description': 'Tools for accessing JPEG2000 files', 'long_description': open('README.md').read(), 'author': 'John Evans', From 371bb90fa22f36e204d28051dcf08d51056a60b2 Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 21 Jul 2013 11:50:07 -0400 Subject: [PATCH 03/30] Qualified on centos 6.4. --- glymur/test/test_opj_suite.py | 4 ++++ release.txt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/glymur/test/test_opj_suite.py b/glymur/test/test_opj_suite.py index 19b0d5a..6e027fa 100644 --- a/glymur/test/test_opj_suite.py +++ b/glymur/test/test_opj_suite.py @@ -7813,6 +7813,8 @@ class TestSuite15(unittest.TestCase): data = jp2.read() self.assertTrue(True) + @unittest.skipIf(glymur.lib.openjpeg.version().startswith('1.3'), + "Segfaults openjpeg 1.3.") def test_NR_DEC_broken2_jp2_5_decode(self): # Null pointer access jfile = os.path.join(data_root, 'input/nonregression/broken2.jp2') @@ -7834,6 +7836,8 @@ class TestSuite15(unittest.TestCase): with self.assertRaises(ValueError) as ce: d = j.read() + @unittest.skipIf(glymur.lib.openjpeg.version().startswith('1.3'), + "Segfaults openjpeg 1.3.") def test_NR_DEC_broken4_jp2_7_decode(self): # Null pointer access jfile = os.path.join(data_root, 'input/nonregression/broken4.jp2') diff --git a/release.txt b/release.txt index 532c3dc..9107589 100644 --- a/release.txt +++ b/release.txt @@ -26,7 +26,7 @@ | Fedora 17 | | X | | Ships with 1.4.0. 169 of 449 tests | | | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ -| CentOS | X | | | Ships with 1.3.0. 167 of 449 tests | +| CentOS | X | | | Ships with 1.3.0. 164 of 450 tests | | 6.3 | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ | Raspberry | | X | | Ships with 1.3.0. 169 of 449 tests | From 162fa8f9c4342e8ec4dea321443b9f81837826e1 Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 21 Jul 2013 12:01:29 -0400 Subject: [PATCH 04/30] Qualified on FC17. --- glymur/test/test_opj_suite.py | 8 ++++---- release.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/glymur/test/test_opj_suite.py b/glymur/test/test_opj_suite.py index 6e027fa..cedd9cf 100644 --- a/glymur/test/test_opj_suite.py +++ b/glymur/test/test_opj_suite.py @@ -7813,8 +7813,8 @@ class TestSuite15(unittest.TestCase): data = jp2.read() self.assertTrue(True) - @unittest.skipIf(glymur.lib.openjpeg.version().startswith('1.3'), - "Segfaults openjpeg 1.3.") + @unittest.skipIf(int(glymur.lib.openjpeg.version().split('.')[1]) < 5, + "Segfaults openjpeg 1.4 and earlier.") def test_NR_DEC_broken2_jp2_5_decode(self): # Null pointer access jfile = os.path.join(data_root, 'input/nonregression/broken2.jp2') @@ -7836,8 +7836,8 @@ class TestSuite15(unittest.TestCase): with self.assertRaises(ValueError) as ce: d = j.read() - @unittest.skipIf(glymur.lib.openjpeg.version().startswith('1.3'), - "Segfaults openjpeg 1.3.") + @unittest.skipIf(int(glymur.lib.openjpeg.version().split('.')[1]) < 5, + "Segfaults openjpeg 1.4 and earlier.") def test_NR_DEC_broken4_jp2_7_decode(self): # Null pointer access jfile = os.path.join(data_root, 'input/nonregression/broken4.jp2') diff --git a/release.txt b/release.txt index 9107589..0ecc20b 100644 --- a/release.txt +++ b/release.txt @@ -23,7 +23,7 @@ | Fedora 18 | | | X | Ships with 1.5.1. 169 of 449 tests | | | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ -| Fedora 17 | | X | | Ships with 1.4.0. 169 of 449 tests | +| Fedora 17 | | X | | Ships with 1.4.0. 166 of 450 tests | | | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ | CentOS | X | | | Ships with 1.3.0. 164 of 450 tests | From de59be99b520af3b683262dd20182ada6c46966f Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 21 Jul 2013 12:13:28 -0400 Subject: [PATCH 05/30] Qualified on raspberry pi. --- release.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.txt b/release.txt index 0ecc20b..c504976 100644 --- a/release.txt +++ b/release.txt @@ -29,7 +29,7 @@ | CentOS | X | | | Ships with 1.3.0. 164 of 450 tests | | 6.3 | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ -| Raspberry | | X | | Ships with 1.3.0. 169 of 449 tests | +| Raspberry | | X | | Ships with 1.3.0. 166 of 450 tests | | Pi | | | | should pass. | | Debian 7 | | | | | +-----------+--------+--------+--------+--------------------------------------+ From b6cb36cb5b72a8d5047277c4f824324defbf4c60 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 21 Jul 2013 12:24:51 -0400 Subject: [PATCH 06/30] Qualified on mac, 2.6, 2.7, 3.3 --- release.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release.txt b/release.txt index c504976..f41ecb4 100644 --- a/release.txt +++ b/release.txt @@ -6,19 +6,19 @@ | | | | | pass. | +-----------+--------+--------+--------+--------------------------------------+ | Mac | X | | | MacPorts with both OpenJPEG 1.5.1 | -| 10.6.8 | | | | and OpenJPEG svn. 353 of 449 tests | +| 10.6.8 | | | | and OpenJPEG svn. 352 of 450 tests | | | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ | Mac | | X | | MacPorts with both OpenJPEG 1.5.1 | -| 10.6.8 | | | | and OpenJPEG svn. 376 of 454 tests | +| 10.6.8 | | | | and OpenJPEG svn. 377 of 455 tests | | | | | | should pass. | +-----------+--------+--------+--------+--------------------------------------+ | Mac | | | X | MacPorts with both OpenJPEG 1.5.1 | -| 10.6.8 | | | | and OpenJPEG svn. 401 of 454 | +| 10.6.8 | | | | and OpenJPEG svn. 402 of 455 | | | | | | tests should pass. | +-----------+--------+--------+-----------------------------------------------+ | Fedora 19 | | | X | Ships with 1.5.1, openjp2 built too. | -| | | | | 401 of 454 tests should pass. | +| | | | | 402 of 455 tests should pass. | +-----------+--------+--------+--------+--------------------------------------+ | Fedora 18 | | | X | Ships with 1.5.1. 169 of 449 tests | | | | | | should pass. | From 64cdbc3ad3ceb3ece3a7854fcabeff652edcbe4f Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 21 Jul 2013 12:31:14 -0400 Subject: [PATCH 07/30] Bumping setup.py to 0.2.4 official. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba8d257..95e778b 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import sys kwargs = {'name': 'Glymur', - 'version': '0.2.4rc1', + 'version': '0.2.4', 'description': 'Tools for accessing JPEG2000 files', 'long_description': open('README.md').read(), 'author': 'John Evans', From efca5e3d342e95515d853196bc5b0af6db014583 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 21 Jul 2013 20:32:18 -0400 Subject: [PATCH 08/30] No need for InconsistentStartOfTileError anymore, closes #81. Also fixed corruption of devel left over from issue number 79. --- glymur/codestream.py | 39 +++++++++-------------------------- glymur/test/test_opj_suite.py | 3 +-- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/glymur/codestream.py b/glymur/codestream.py index d8ef0e2..df1575c 100644 --- a/glymur/codestream.py +++ b/glymur/codestream.py @@ -5,7 +5,6 @@ codestreams. """ # pylint: disable=C0302,R0902,R0903,R0913 -from itertools import takewhile import math import struct import sys @@ -43,16 +42,6 @@ for _marker in range(0xff90, 0xff94): _VALID_MARKERS.append(_marker) -class InconsistentStartOfTileError(IOError): - """To be raised if bad SOT segment encountered. - - SOT segment offsets are recorded as encountered. The offsets should all be - different. - """ - def __init__(self, msg): - IOError.__init__(self, msg) - - class Codestream(object): """Container for codestream information. @@ -115,6 +104,11 @@ class Codestream(object): read_buffer = fptr.read(2) marker_id, = struct.unpack('>H', read_buffer) + if marker_id == 0xff90 and header_only: + # Start-of-tile (SOT) means that we are out of the main header + # and there is no need to go further. + break + try: segment = self._process_marker_segment(fptr, marker_id) except Exception as error: @@ -139,12 +133,6 @@ class Codestream(object): fptr.seek(self._tile_offset[-1] + self._tile_length[-1]) - if header_only: - # start-of-tile (SOT) means we are out of the main header. - # No need to go any further. - gen = takewhile(lambda s: s.marker_id != 'SOT', self.segment) - self.segment = list(gen) - def _process_marker_segment(self, fptr, marker_id): """Process and return a segment from the codestream. @@ -210,19 +198,12 @@ class Codestream(object): # we encounter start-of-data marker segments. segment = _parse_sot_segment(fptr) - if segment.offset not in self._tile_offset: - self._tile_offset.append(segment.offset) - if segment.psot == 0: - tile_part_length = self.offset + self.length - segment.offset - 2 - else: - tile_part_length = segment.psot - self._tile_length.append(tile_part_length) + self._tile_offset.append(segment.offset) + if segment.psot == 0: + tile_part_length = self.offset + self.length - segment.offset - 2 else: - msg = "Inconsistent start-of-tile (SOT) marker segment " - msg += "encountered in tile with index {0}. " - msg += "Codestream parsing terminated." - msg = msg.format(segment.isot) - raise InconsistentStartOfTileError(msg) + tile_part_length = segment.psot + self._tile_length.append(tile_part_length) elif marker_id == 0xff93: # start of data. Need to seek past the current tile part. diff --git a/glymur/test/test_opj_suite.py b/glymur/test/test_opj_suite.py index cedd9cf..0371acf 100644 --- a/glymur/test/test_opj_suite.py +++ b/glymur/test/test_opj_suite.py @@ -6954,8 +6954,7 @@ class TestSuiteDump(unittest.TestCase): self.assertEqual(c.segment[3]._exponent, [4] + [5, 5, 6] * 5) def test_NR_merged_dump(self): - jfile = os.path.join(data_root, - 'input/nonregression/merged.jp2') + jfile = os.path.join(data_root, 'input/nonregression/merged.jp2') jp2 = Jp2k(jfile) ids = [box.box_id for box in jp2.box] From 769dc23cdc278c06e2335bf5acd407d9516ec449 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 22 Jul 2013 08:36:44 -0400 Subject: [PATCH 09/30] XML entities to be stored as ElementTree objects, not as "Elements". #82 --- glymur/jp2box.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index d205e1b..15471a5 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1608,7 +1608,7 @@ class XMLBox(Jp2kBox): offset of the box from the start of the file. longname : str more verbose description of the box. - xml : ElementTree.Element + xml : ElementTree object XML section. """ def __init__(self, xml=None, filename=None, length=0, offset=-1): @@ -1636,10 +1636,7 @@ class XMLBox(Jp2kBox): msg = Jp2kBox.__str__(self) xml = self.xml if self.xml is not None: - try: - msg += _pretty_print_xml(self.xml) - except TypeError: - msg += _pretty_print_xml(self.xml.getroot()) + msg += _pretty_print_xml(self.xml) else: msg += '\n {0}'.format(xml) return msg @@ -1682,7 +1679,8 @@ class XMLBox(Jp2kBox): text = text.rstrip('\0') try: - xml = ET.fromstring(text) + elt = ET.fromstring(text) + xml = ET.ElementTree(elt) except ParseError as parse_error: msg = 'A problem was encountered while parsing an XML box: "{0}"' msg = msg.format(str(parse_error)) @@ -1926,13 +1924,11 @@ class UUIDBox(Jp2kBox): # XMP data. Parse as XML. Seems to be a difference between # ElementTree in version 2.7 and 3.3. if sys.hexversion < 0x03000000: - #parser = ET.XMLParser(encoding='utf-8') - #import pdb; pdb.set_trace() - #self.data = ET.fromstringlist(raw_data, parser=parser) - self.data = ET.fromstring(raw_data) + elt = ET.fromstring(raw_data) else: text = raw_data.decode('utf-8') - self.data = ET.fromstring(text) + elt = ET.fromstring(text) + self.data = ET.ElementTree(elt) elif the_uuid.bytes == b'JpgTiffExif->JP2': exif_obj = Exif(raw_data) ifds = OrderedDict() @@ -2555,8 +2551,8 @@ def _pretty_print_xml(xml, level=0): """Pretty print XML data. """ xml = copy.deepcopy(xml) - _indent(xml, level=level) - xmltext = ET.tostring(xml).decode('utf-8') + _indent(xml.getroot(), level=level) + xmltext = ET.tostring(xml.getroot()).decode('utf-8') # Indent it a bit. lst = [(' ' + x) for x in xmltext.split('\n')] From 2d38222117088493fc5b522df0625d92910c5297 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 22 Jul 2013 08:37:32 -0400 Subject: [PATCH 10/30] Updated tests to work with ElementTree objects only. Closes #82. --- glymur/test/test_jp2box.py | 2 +- glymur/test/test_opj_suite.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 8fa566e..178d4a0 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -365,7 +365,7 @@ class TestXML(unittest.TestCase): j2k.wrap(tfile.name, boxes=boxes) jp2 = Jp2k(tfile.name) self.assertEqual(jp2.box[3].box_id, 'xml ') - self.assertEqual(ET.tostring(jp2.box[3].xml), + self.assertEqual(ET.tostring(jp2.box[3].xml.getroot()), b'0') @unittest.skipIf(os.name == "nt", diff --git a/glymur/test/test_opj_suite.py b/glymur/test/test_opj_suite.py index 0371acf..4aab788 100644 --- a/glymur/test/test_opj_suite.py +++ b/glymur/test/test_opj_suite.py @@ -4021,7 +4021,7 @@ class TestSuiteDump(unittest.TestCase): self.assertEqual(jp2.box[1].compatibility_list[1], 'jp2 ') # XML box - tags = [x.tag for x in jp2.box[2].xml] + tags = [x.tag for x in jp2.box[2].xml.getroot()] self.assertEqual(tags, ['{http://www.jpeg.org/jpx/1.0/xml}' + 'GENERAL_CREATION_INFO']) @@ -4046,7 +4046,7 @@ class TestSuiteDump(unittest.TestCase): self.assertEqual(jp2.box[3].box[1].colorspace, glymur.core.SRGB) # XML box - tags = [x.tag for x in jp2.box[4].xml] + tags = [x.tag for x in jp2.box[4].xml.getroot()] self.assertEqual(tags, ['{http://www.jpeg.org/jpx/1.0/xml}CAPTION', '{http://www.jpeg.org/jpx/1.0/xml}LOCATION', '{http://www.jpeg.org/jpx/1.0/xml}EVENT']) @@ -4376,13 +4376,13 @@ class TestSuiteDump(unittest.TestCase): self.assertIsNone(jp2.box[2].box[1].colorspace) # XML box - tags = [x.tag for x in jp2.box[3].xml] + tags = [x.tag for x in jp2.box[3].xml.getroot()] self.assertEqual(tags, ['{http://www.jpeg.org/jpx/1.0/xml}' + 'GENERAL_CREATION_INFO']) # XML box - tags = [x.tag for x in jp2.box[5].xml] + tags = [x.tag for x in jp2.box[5].xml.getroot()] self.assertEqual(tags, ['{http://www.jpeg.org/jpx/1.0/xml}CAPTION', '{http://www.jpeg.org/jpx/1.0/xml}LOCATION', From d0eeb26748211524d2be1511855869685de6d58e Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 22 Jul 2013 08:47:59 -0400 Subject: [PATCH 11/30] No longer need to mention thought of going to lxml. Closes #65. All we have to do is to be consistent in going forward with usage of ElementTree. We should store ElementTree instances, not "Element" instances. See issue 82. --- docs/source/roadmap.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst index ea4a5bd..f1434b2 100644 --- a/docs/source/roadmap.rst +++ b/docs/source/roadmap.rst @@ -6,7 +6,6 @@ Here's an incomplete list of what I'd like to focus on in the near future. * continue to monitor upstream changes in the openjp2 library * investigate using CFFI or cython instead of ctypes to wrap openjp2 - * investigate swapping out ElementTree for LXML * eventually expose the openjp2 API * investigate JPIP From 11eed0f513b707ee940751132584f6b183496592 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 17:49:50 -0400 Subject: [PATCH 12/30] Prepping for 0.2.5 release. --- CHANGES.txt | 3 +++ docs/source/conf.py | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f80b04b..410f243 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +Jul 23, 2013 - v0.2.5 Fixed inconsistency in XML handling, now all instances + are always ElementTree objects (issue82). + Jul 21, 2013 - v0.2.4 Fixed markdown bug for Fedora 17 information, fixed out-of-date windows information (issue79). Fixed incorrect interpretation of Psot parameter (issue78). diff --git a/docs/source/conf.py b/docs/source/conf.py index bd59b1b..d9f833e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,7 +78,7 @@ copyright = u'2013, John Evans' # The short X.Y version. version = '0.1' # The full version, including alpha/beta/rc tags. -release = '0.2.4' +release = '0.2.5' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 95e778b..4f33418 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import sys kwargs = {'name': 'Glymur', - 'version': '0.2.4', + 'version': '0.2.5rc1', 'description': 'Tools for accessing JPEG2000 files', 'long_description': open('README.md').read(), 'author': 'John Evans', From e75cdcf68665c1a1ef47220ba32205241e2ff38b Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 18:00:44 -0400 Subject: [PATCH 13/30] The XMP uuid data is an ElementTree instance, not an Element instance. --- docs/source/how_do_i.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index 8abb693..9ffaee5 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -185,9 +185,8 @@ Work with XMP UUIDs? ==================== The example JP2 file shipped with glymur has an XMP UUID. :: - >>> from glymur import Jp2k - >>> file = glymur.data.nemo() - >>> j = Jp2k(file) + >>> import glymur + >>> j = glymur.Jp2k(glymur.data.nemo()) >>> print(j.box[4]) UUID Box (uuid) @ (715, 2412) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) @@ -198,7 +197,7 @@ The example JP2 file shipped with glymur has an XMP UUID. :: -Since the UUID data in this case is returned as an ElementTree Element, one can +Since the UUID data in this case is returned as an ElementTree instance, one can use ElementTree to access the data. For example, to extract the **CreatorTool** attribute value, the following would work:: From 49634552986dc25c4c731ed00cad88fb4ff7741f Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 18:07:17 -0400 Subject: [PATCH 14/30] Finalizing 0.2.5 version info. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f33418..8d88a3a 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages import sys kwargs = {'name': 'Glymur', - 'version': '0.2.5rc1', + 'version': '0.2.5', 'description': 'Tools for accessing JPEG2000 files', 'long_description': open('README.md').read(), 'author': 'John Evans', From 83f17b218114c637eefec1c51a1ac55bc7b62f8a Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 19:03:59 -0400 Subject: [PATCH 15/30] Checking for the codestream markers to always occupy two bytes. --- docs/source/detailed_installation.rst | 13 +++++-- glymur/codestream.py | 9 ++++- glymur/test/test_format_corpus.py | 51 +++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 glymur/test/test_format_corpus.py diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index 06c6b79..b737843 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -137,8 +137,17 @@ platforms. Testing ''''''' -If you wish to run the tests (strongly recommended :-), you can either run them -from within python as follows ... :: +There are two environment variables you may wish to set before running the +tests. + + * **OPJ_DATA_ROOT** - points to directory for OpenJPEG test data + * **FORMAT_CORPUS_ROOT** - points to directory for format-corpus repository (see https://github.com/openplanets/format-corpus) + +Setting these two environment variables is not required, as any tests using +either of them will be skipped. + +In order to run the tests, you can either run them from within +python as follows ... :: >>> import glymur >>> glymur.runtests() diff --git a/glymur/codestream.py b/glymur/codestream.py index df1575c..c4d8b4b 100644 --- a/glymur/codestream.py +++ b/glymur/codestream.py @@ -102,7 +102,14 @@ class Codestream(object): while True: read_buffer = fptr.read(2) - marker_id, = struct.unpack('>H', read_buffer) + try: + marker_id, = struct.unpack('>H', read_buffer) + except struct.error: + # Treat this as a warning. + msg = "Marker had length {0} instead of expected length of 2 " + msg += "bytes. Codestream parsing terminated." + warnings.warn(msg.format(len(read_buffer))) + break if marker_id == 0xff90 and header_only: # Start-of-tile (SOT) means that we are out of the main header diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py new file mode 100644 index 0000000..2f7347b --- /dev/null +++ b/glymur/test/test_format_corpus.py @@ -0,0 +1,51 @@ +""" +These tests deal with JPX/JP2/J2K images in the format-corpus repository. +""" +#pylint: disable-all + +import os +import sys + +if sys.hexversion < 0x02070000: + import unittest2 as unittest +else: + import unittest + +import warnings + +from glymur import Jp2k +import glymur + +try: + data_root = os.environ['FORMAT_CORPUS_ROOT'] +except KeyError: + data_root = None +except: + raise + + +@unittest.skipIf(sys.hexversion < 0x03020000, + "Requires features introduced in 3.2 (assertWarns)") +@unittest.skipIf(data_root is None, + "FORMAT_CORPUS_ROOT environment variable not set") +class TestSuite(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_balloon_trunc1(self): + # Has one byte shaved off of EOC marker. + jfile = os.path.join(data_root, + 'jp2k-test/byteCorruption/balloon_trunc1.jp2') + j2k = Jp2k(jfile) + with self.assertWarns(UserWarning): + c = j2k.get_codestream(header_only=False) + + # The last segment is truncated, so there should not be an EOC marker. + self.assertNotEqual(c.segment[-1].marker_id, 'EOC') + +if __name__ == "__main__": + unittest.main() From 15cd084df132aef5245066d80105f950043cbd5a Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 19:03:59 -0400 Subject: [PATCH 16/30] Checking for the codestream markers to always occupy two bytes. #77 --- docs/source/detailed_installation.rst | 13 +++++-- glymur/codestream.py | 9 ++++- glymur/test/test_format_corpus.py | 51 +++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) create mode 100644 glymur/test/test_format_corpus.py diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index 06c6b79..b737843 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -137,8 +137,17 @@ platforms. Testing ''''''' -If you wish to run the tests (strongly recommended :-), you can either run them -from within python as follows ... :: +There are two environment variables you may wish to set before running the +tests. + + * **OPJ_DATA_ROOT** - points to directory for OpenJPEG test data + * **FORMAT_CORPUS_ROOT** - points to directory for format-corpus repository (see https://github.com/openplanets/format-corpus) + +Setting these two environment variables is not required, as any tests using +either of them will be skipped. + +In order to run the tests, you can either run them from within +python as follows ... :: >>> import glymur >>> glymur.runtests() diff --git a/glymur/codestream.py b/glymur/codestream.py index df1575c..c4d8b4b 100644 --- a/glymur/codestream.py +++ b/glymur/codestream.py @@ -102,7 +102,14 @@ class Codestream(object): while True: read_buffer = fptr.read(2) - marker_id, = struct.unpack('>H', read_buffer) + try: + marker_id, = struct.unpack('>H', read_buffer) + except struct.error: + # Treat this as a warning. + msg = "Marker had length {0} instead of expected length of 2 " + msg += "bytes. Codestream parsing terminated." + warnings.warn(msg.format(len(read_buffer))) + break if marker_id == 0xff90 and header_only: # Start-of-tile (SOT) means that we are out of the main header diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py new file mode 100644 index 0000000..2f7347b --- /dev/null +++ b/glymur/test/test_format_corpus.py @@ -0,0 +1,51 @@ +""" +These tests deal with JPX/JP2/J2K images in the format-corpus repository. +""" +#pylint: disable-all + +import os +import sys + +if sys.hexversion < 0x02070000: + import unittest2 as unittest +else: + import unittest + +import warnings + +from glymur import Jp2k +import glymur + +try: + data_root = os.environ['FORMAT_CORPUS_ROOT'] +except KeyError: + data_root = None +except: + raise + + +@unittest.skipIf(sys.hexversion < 0x03020000, + "Requires features introduced in 3.2 (assertWarns)") +@unittest.skipIf(data_root is None, + "FORMAT_CORPUS_ROOT environment variable not set") +class TestSuite(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_balloon_trunc1(self): + # Has one byte shaved off of EOC marker. + jfile = os.path.join(data_root, + 'jp2k-test/byteCorruption/balloon_trunc1.jp2') + j2k = Jp2k(jfile) + with self.assertWarns(UserWarning): + c = j2k.get_codestream(header_only=False) + + # The last segment is truncated, so there should not be an EOC marker. + self.assertNotEqual(c.segment[-1].marker_id, 'EOC') + +if __name__ == "__main__": + unittest.main() From 67032f504071c2a491836180401af6ad0c0b4177 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 19:13:26 -0400 Subject: [PATCH 17/30] Added test for error on truncated EOC file. --- glymur/test/test_format_corpus.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py index 2f7347b..fdf042c 100644 --- a/glymur/test/test_format_corpus.py +++ b/glymur/test/test_format_corpus.py @@ -47,5 +47,9 @@ class TestSuite(unittest.TestCase): # The last segment is truncated, so there should not be an EOC marker. self.assertNotEqual(c.segment[-1].marker_id, 'EOC') + # The codestream is not as long as claimed. + with self.assertRaises(OSError): + j2k.read(rlevel=-1) + if __name__ == "__main__": unittest.main() From 292f806bfeb78936692008b65be69d72ab3d9c51 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 22 Jul 2013 19:28:51 -0400 Subject: [PATCH 18/30] Added test for file shortened by 5000 bytes. --- glymur/test/test_format_corpus.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py index fdf042c..14c0830 100644 --- a/glymur/test/test_format_corpus.py +++ b/glymur/test/test_format_corpus.py @@ -51,5 +51,20 @@ class TestSuite(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) + def test_balloon_trunc2(self): + # Shortened by 5000 bytes. + jfile = os.path.join(data_root, + 'jp2k-test/byteCorruption/balloon_trunc2.jp2') + j2k = Jp2k(jfile) + with self.assertWarns(UserWarning): + c = j2k.get_codestream(header_only=False) + + # The last segment is truncated, so there should not be an EOC marker. + self.assertNotEqual(c.segment[-1].marker_id, 'EOC') + + # The codestream is not as long as claimed. + with self.assertRaises(OSError): + j2k.read(rlevel=-1) + if __name__ == "__main__": unittest.main() From 83b53c5f9be71139ad2320edab53ee26d8110639 Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 23 Jul 2013 06:36:36 -0400 Subject: [PATCH 19/30] Added warning test for image with last truncated tile, #77 --- glymur/test/test_format_corpus.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py index 14c0830..28e1d4d 100644 --- a/glymur/test/test_format_corpus.py +++ b/glymur/test/test_format_corpus.py @@ -66,5 +66,20 @@ class TestSuite(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) + def test_balloon_trunc3(self): + # Most of last tile is missing. + jfile = os.path.join(data_root, + 'jp2k-test/byteCorruption/balloon_trunc3.jp2') + j2k = Jp2k(jfile) + with self.assertWarns(UserWarning): + c = j2k.get_codestream(header_only=False) + + # The last segment is truncated, so there should not be an EOC marker. + self.assertNotEqual(c.segment[-1].marker_id, 'EOC') + + # Should error out, it does not. + #with self.assertRaises(OSError): + # j2k.read(rlevel=-1) + if __name__ == "__main__": unittest.main() From de7e6e1f0fef2585e128ca9f5ba5ea195c410179 Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 23 Jul 2013 17:22:21 -0400 Subject: [PATCH 20/30] Added validation step for jp2 brand / any icc profile mismatch. #77 text_GBR.jp2 also shows this issue. --- glymur/__init__.py | 2 +- glymur/jp2k.py | 18 ++++++++++++++++++ glymur/test/test_format_corpus.py | 11 +++++++++++ glymur/test/test_opj_suite.py | 11 +++++++++-- glymur/test/test_printing.py | 22 ++++++++++++++++------ 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/glymur/__init__.py b/glymur/__init__.py index a69d28f..fb37ef0 100644 --- a/glymur/__init__.py +++ b/glymur/__init__.py @@ -6,7 +6,7 @@ from .jp2k import Jp2k from .jp2dump import jp2dump from . import data -from . import test +#from . import test def runtests(): diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 5baebac..4efcb77 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -22,6 +22,7 @@ from .codestream import Codestream from .core import SRGB from .core import GREYSCALE from .core import PROGRESSION_ORDER +from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .jp2box import Jp2kBox from .jp2box import JPEG2000SignatureBox from .jp2box import FileTypeBox @@ -159,6 +160,23 @@ class Jp2k(Jp2kBox): # boxes) here. fptr.seek(0) self.box = self.parse_superbox(fptr) + self._validate() + + def _validate(self): + """Validate the JPEG 2000 outermost superbox. + """ + # A jp2-branded file cannot contain an "any ICC profile + ftyp = self.box[1] + jp2h = [box for box in self.box if box.box_id == 'jp2h'][0] + colr = [box for box in jp2h.box if box.box_id == 'colr'][0] + if (((ftyp.brand == 'jp2 ') and + (colr.method not in (ENUMERATED_COLORSPACE, + RESTRICTED_ICC_PROFILE)))): + msg = "Color Specification box method must specify either an " + msg += "enumerated colorspace or a restricted ICC profile if the " + msg += "file type box brand is 'jp2 '." + warnings.warn(msg) + # pylint: disable-msg=W0221 def write(self, img_array, cratios=None, eph=False, psnr=None, numres=None, diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py index 28e1d4d..93af29d 100644 --- a/glymur/test/test_format_corpus.py +++ b/glymur/test/test_format_corpus.py @@ -81,5 +81,16 @@ class TestSuite(unittest.TestCase): #with self.assertRaises(OSError): # j2k.read(rlevel=-1) + def test_jp2_brand_vs_any_icc_profile(self): + # If 'jp2 ', then the method cannot be any icc profile. + jfile = os.path.join(data_root, + 'jp2k-test/icc/balloon_eciRGBv2_ps_adobeplugin.jpf') + with self.assertWarns(UserWarning): + j2k = Jp2k(jfile) + + # Should error out, it does not. + #with self.assertRaises(OSError): + # j2k.read(rlevel=-1) + if __name__ == "__main__": unittest.main() diff --git a/glymur/test/test_opj_suite.py b/glymur/test/test_opj_suite.py index 4aab788..4699b06 100644 --- a/glymur/test/test_opj_suite.py +++ b/glymur/test/test_opj_suite.py @@ -990,7 +990,11 @@ class TestSuite(unittest.TestCase): def test_NR_DEC_text_GBR_jp2_29_decode(self): jfile = os.path.join(data_root, 'input/nonregression/text_GBR.jp2') - data = Jp2k(jfile).read() + with warnings.catch_warnings(): + # brand is 'jp2 ', but has any icc profile. + warnings.simplefilter("ignore") + jp2 = Jp2k(jfile) + data = jp2.read() self.assertTrue(True) def test_NR_DEC_pacs_ge_j2k_30_decode(self): @@ -7260,7 +7264,10 @@ class TestSuiteDump(unittest.TestCase): def test_NR_text_GBR_dump(self): jfile = os.path.join(data_root, 'input/nonregression/text_GBR.jp2') - jp2 = Jp2k(jfile) + with warnings.catch_warnings(): + # brand is 'jp2 ', but has any icc profile. + warnings.simplefilter("ignore") + jp2 = Jp2k(jfile) ids = [box.box_id for box in jp2.box] lst = ['jP ', 'ftyp', 'rreq', 'jp2h', diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 09f828c..79a12e1 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -4,6 +4,7 @@ import pkg_resources import struct import sys import tempfile +import warnings if sys.hexversion < 0x02070000: import unittest2 as unittest @@ -277,9 +278,12 @@ class TestPrinting(unittest.TestCase): "OPJ_DATA_ROOT environment variable not set") def test_icc_profile(self): filename = os.path.join(data_root, 'input/nonregression/text_GBR.jp2') - j = glymur.Jp2k(filename) + with warnings.catch_warnings(): + # brand is 'jp2 ', but has any icc profile. + warnings.simplefilter("ignore") + jp2 = Jp2k(filename) with patch('sys.stdout', new=StringIO()) as fake_out: - print(j.box[3].box[1]) + print(jp2.box[3].box[1]) actual = fake_out.getvalue().strip() lin27 = ["Colour Specification Box (colr) @ (179, 1339)", " Method: any ICC profile", @@ -902,10 +906,13 @@ class TestPrinting(unittest.TestCase): # ICC profiles may be used in JP2, but the approximation field should # be zero unless we have jpx. This file does both. filename = os.path.join(data_root, 'input/nonregression/text_GBR.jp2') - j = glymur.Jp2k(filename) + with warnings.catch_warnings(): + # brand is 'jp2 ', but has any icc profile. + warnings.simplefilter("ignore") + jp2 = Jp2k(filename) with patch('sys.stdout', new=StringIO()) as fake_out: - print(j.box[3].box[1]) + print(jp2.box[3].box[1]) actual = fake_out.getvalue().strip() lines = ["Colour Specification Box (colr) @ (179, 1339)", " Method: any ICC profile", @@ -942,10 +949,13 @@ class TestPrinting(unittest.TestCase): def test_uuid(self): # UUID box filename = os.path.join(data_root, 'input/nonregression/text_GBR.jp2') - j = glymur.Jp2k(filename) + with warnings.catch_warnings(): + # brand is 'jp2 ', but has any icc profile. + warnings.simplefilter("ignore") + jp2 = Jp2k(filename) with patch('sys.stdout', new=StringIO()) as fake_out: - print(j.box[4]) + print(jp2.box[4]) actual = fake_out.getvalue().strip() lines = ['UUID Box (uuid) @ (1544, 25)', ' UUID: 3a0d0218-0ae9-4115-b376-4bca41ce0e71', From 6fd8f6c3b839665a65d822b2d6e40733f6f371f5 Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 23 Jul 2013 18:27:09 -0400 Subject: [PATCH 21/30] text_GBR also has a jp2 brand any icc profile mismatch. #77 --- glymur/test/test_format_corpus.py | 56 ++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_format_corpus.py index 93af29d..f9b94e9 100644 --- a/glymur/test/test_format_corpus.py +++ b/glymur/test/test_format_corpus.py @@ -17,18 +17,23 @@ from glymur import Jp2k import glymur try: - data_root = os.environ['FORMAT_CORPUS_ROOT'] + format_corpus_data_root = os.environ['FORMAT_CORPUS_DATA_ROOT'] except KeyError: - data_root = None + format_corpus_data_root = None +except: + raise + +try: + opj_data_root = os.environ['OPJ_DATA_ROOT'] +except KeyError: + opj_corpus_data_root = None except: raise @unittest.skipIf(sys.hexversion < 0x03020000, "Requires features introduced in 3.2 (assertWarns)") -@unittest.skipIf(data_root is None, - "FORMAT_CORPUS_ROOT environment variable not set") -class TestSuite(unittest.TestCase): +class TestSuiteFormatCorpus(unittest.TestCase): def setUp(self): pass @@ -36,9 +41,11 @@ class TestSuite(unittest.TestCase): def tearDown(self): pass + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc1(self): # Has one byte shaved off of EOC marker. - jfile = os.path.join(data_root, + jfile = os.path.join(format_corpus_data_root, 'jp2k-test/byteCorruption/balloon_trunc1.jp2') j2k = Jp2k(jfile) with self.assertWarns(UserWarning): @@ -51,9 +58,11 @@ class TestSuite(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc2(self): # Shortened by 5000 bytes. - jfile = os.path.join(data_root, + jfile = os.path.join(format_corpus_data_root, 'jp2k-test/byteCorruption/balloon_trunc2.jp2') j2k = Jp2k(jfile) with self.assertWarns(UserWarning): @@ -66,9 +75,11 @@ class TestSuite(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc3(self): # Most of last tile is missing. - jfile = os.path.join(data_root, + jfile = os.path.join(format_corpus_data_root, 'jp2k-test/byteCorruption/balloon_trunc3.jp2') j2k = Jp2k(jfile) with self.assertWarns(UserWarning): @@ -81,16 +92,35 @@ class TestSuite(unittest.TestCase): #with self.assertRaises(OSError): # j2k.read(rlevel=-1) + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_jp2_brand_vs_any_icc_profile(self): # If 'jp2 ', then the method cannot be any icc profile. - jfile = os.path.join(data_root, - 'jp2k-test/icc/balloon_eciRGBv2_ps_adobeplugin.jpf') + jfile = os.path.join(format_corpus_data_root, + 'jp2k-test', 'icc', + 'balloon_eciRGBv2_ps_adobeplugin.jpf') with self.assertWarns(UserWarning): j2k = Jp2k(jfile) - # Should error out, it does not. - #with self.assertRaises(OSError): - # j2k.read(rlevel=-1) +@unittest.skipIf(sys.hexversion < 0x03020000, + "Requires features introduced in 3.2 (assertWarns)") +class TestSuiteOpj(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @unittest.skipIf(opj_data_root is None, + "OPJ_DATA_ROOT environment variable not set") + def test_jp2_brand_vs_any_icc_profile(self): + # If 'jp2 ', then the method cannot be any icc profile. + filename = os.path.join(opj_data_root, + 'input/nonregression/text_GBR.jp2') + with self.assertWarns(UserWarning): + j2k = Jp2k(filename) + if __name__ == "__main__": unittest.main() From 097549fd667305f4b9e3c03fc0a208b615742541 Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 23 Jul 2013 18:39:42 -0400 Subject: [PATCH 22/30] Renamed, files other than those from format-corpus. #77 --- glymur/test/{test_format_corpus.py => test_conformance.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename glymur/test/{test_format_corpus.py => test_conformance.py} (100%) diff --git a/glymur/test/test_format_corpus.py b/glymur/test/test_conformance.py similarity index 100% rename from glymur/test/test_format_corpus.py rename to glymur/test/test_conformance.py From ee2b3b9c0e61097fa68502f86874611366267fda Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 06:01:46 -0400 Subject: [PATCH 23/30] Validation check now allows for multiple colr boxes. Closes #77 jp2h superboxes can contain multiple colr boxes, so we need to check each one for validity. --- glymur/jp2k.py | 19 ++++++++++--------- glymur/test/test_conformance.py | 29 ++++++++++++++++++----------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 4efcb77..319bf46 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -167,15 +167,16 @@ class Jp2k(Jp2kBox): """ # A jp2-branded file cannot contain an "any ICC profile ftyp = self.box[1] - jp2h = [box for box in self.box if box.box_id == 'jp2h'][0] - colr = [box for box in jp2h.box if box.box_id == 'colr'][0] - if (((ftyp.brand == 'jp2 ') and - (colr.method not in (ENUMERATED_COLORSPACE, - RESTRICTED_ICC_PROFILE)))): - msg = "Color Specification box method must specify either an " - msg += "enumerated colorspace or a restricted ICC profile if the " - msg += "file type box brand is 'jp2 '." - warnings.warn(msg) + if ftyp.brand == 'jp2 ': + jp2h = [box for box in self.box if box.box_id == 'jp2h'][0] + colrs = [box for box in jp2h.box if box.box_id == 'colr'] + for colr in colrs: + if colr.method not in (ENUMERATED_COLORSPACE, + RESTRICTED_ICC_PROFILE): + msg = "Color Specification box method must specify either " + msg += "an enumerated colorspace or a restricted ICC " + msg += "profile if the file type box brand is 'jp2 '." + warnings.warn(msg) # pylint: disable-msg=W0221 diff --git a/glymur/test/test_conformance.py b/glymur/test/test_conformance.py index f9b94e9..ec179c2 100644 --- a/glymur/test/test_conformance.py +++ b/glymur/test/test_conformance.py @@ -26,11 +26,13 @@ except: try: opj_data_root = os.environ['OPJ_DATA_ROOT'] except KeyError: - opj_corpus_data_root = None + opj_data_root = None except: raise +@unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") @unittest.skipIf(sys.hexversion < 0x03020000, "Requires features introduced in 3.2 (assertWarns)") class TestSuiteFormatCorpus(unittest.TestCase): @@ -41,8 +43,6 @@ class TestSuiteFormatCorpus(unittest.TestCase): def tearDown(self): pass - @unittest.skipIf(format_corpus_data_root is None, - "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc1(self): # Has one byte shaved off of EOC marker. jfile = os.path.join(format_corpus_data_root, @@ -58,8 +58,6 @@ class TestSuiteFormatCorpus(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) - @unittest.skipIf(format_corpus_data_root is None, - "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc2(self): # Shortened by 5000 bytes. jfile = os.path.join(format_corpus_data_root, @@ -75,8 +73,6 @@ class TestSuiteFormatCorpus(unittest.TestCase): with self.assertRaises(OSError): j2k.read(rlevel=-1) - @unittest.skipIf(format_corpus_data_root is None, - "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_balloon_trunc3(self): # Most of last tile is missing. jfile = os.path.join(format_corpus_data_root, @@ -92,8 +88,6 @@ class TestSuiteFormatCorpus(unittest.TestCase): #with self.assertRaises(OSError): # j2k.read(rlevel=-1) - @unittest.skipIf(format_corpus_data_root is None, - "FORMAT_CORPUS_DATA_ROOT environment variable not set") def test_jp2_brand_vs_any_icc_profile(self): # If 'jp2 ', then the method cannot be any icc profile. jfile = os.path.join(format_corpus_data_root, @@ -102,7 +96,22 @@ class TestSuiteFormatCorpus(unittest.TestCase): with self.assertWarns(UserWarning): j2k = Jp2k(jfile) + def test_jp2_brand_vs_any_icc_profile_multiple_colr(self): + # Has colr box, one that conforms, one that does not. + # Wrong 'brand' field; contains two versions of ICC profile: one + # embedded using "Any ICC" method; other embedded using "Restricted + # ICC" method, with description ("Modified eciRGB v2") and profileClass + # ("Input Device") changed relative to original profile. + lst = [format_corpus_data_root, 'jp2k-test', 'icc', + 'balloon_eciRGBv2_ps_adobeplugin_jp2compatible.jpf'] + jfile = os.path.join(*lst) + with self.assertWarns(UserWarning): + j2k = Jp2k(jfile) + + +@unittest.skipIf(opj_data_root is None, + "OPJ_DATA_ROOT environment variable not set") @unittest.skipIf(sys.hexversion < 0x03020000, "Requires features introduced in 3.2 (assertWarns)") class TestSuiteOpj(unittest.TestCase): @@ -113,8 +122,6 @@ class TestSuiteOpj(unittest.TestCase): def tearDown(self): pass - @unittest.skipIf(opj_data_root is None, - "OPJ_DATA_ROOT environment variable not set") def test_jp2_brand_vs_any_icc_profile(self): # If 'jp2 ', then the method cannot be any icc profile. filename = os.path.join(opj_data_root, From f8406fc20a5ac0f0c28cd74b46ba610a3739e085 Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 11:59:45 -0400 Subject: [PATCH 24/30] No real need to import the test packages. Discovery works just fine. #84 --- glymur/__init__.py | 1 - glymur/test/__init__.py | 11 ----------- glymur/test/test_conformance.py | 4 ---- 3 files changed, 16 deletions(-) diff --git a/glymur/__init__.py b/glymur/__init__.py index fb37ef0..fb1c504 100644 --- a/glymur/__init__.py +++ b/glymur/__init__.py @@ -6,7 +6,6 @@ from .jp2k import Jp2k from .jp2dump import jp2dump from . import data -#from . import test def runtests(): diff --git a/glymur/test/__init__.py b/glymur/test/__init__.py index 89c7eea..e69de29 100644 --- a/glymur/test/__init__.py +++ b/glymur/test/__init__.py @@ -1,11 +0,0 @@ -from .test_callbacks import TestCallbacks as callbacks -from .test_codestream import TestCodestream as codestream -from .test_config import TestSuite as config -from .test_jp2k import TestJp2k as jp2k -from .test_icc import TestICC as icc -from .test_printing import TestPrinting as printing -from .test_opj_suite import TestSuite as suite -from .test_opj_suite import TestSuiteDump as suitedump -from .test_opj_suite_write import TestSuiteWrite as suitew -from .test_opj_suite_neg import TestSuiteNegative as suiteneg -from .test_jp2box import TestJp2Boxes as box diff --git a/glymur/test/test_conformance.py b/glymur/test/test_conformance.py index ec179c2..9f40b3e 100644 --- a/glymur/test/test_conformance.py +++ b/glymur/test/test_conformance.py @@ -20,15 +20,11 @@ try: format_corpus_data_root = os.environ['FORMAT_CORPUS_DATA_ROOT'] except KeyError: format_corpus_data_root = None -except: - raise try: opj_data_root = os.environ['OPJ_DATA_ROOT'] except KeyError: opj_data_root = None -except: - raise @unittest.skipIf(format_corpus_data_root is None, From 0edbbfd2e1860b46a8a102512b55d274adc04691 Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 12:02:29 -0400 Subject: [PATCH 25/30] Added tests for jpch, jplh superboxes. Closes #84 --- glymur/jp2box.py | 195 ++++++++++++++++++++++++++++++++++++- glymur/test/test_jp2box.py | 39 ++++++++ 2 files changed, 232 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 15471a5..b5637f6 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -520,6 +520,195 @@ class ChannelDefinitionBox(Jp2kBox): return box +class CodestreamHeaderBox(Jp2kBox): + """Container for codestream header box information. + + Attributes + ---------- + box_id : str + 4-character identifier for the box. + length : int + length of the box in bytes. + offset : int + offset of the box from the start of the file. + longname : str + more verbose description of the box. + box : list + List of boxes contained in this superbox. + """ + def __init__(self, length=0, offset=-1): + Jp2kBox.__init__(self, box_id='jpch', longname='Codestream Header') + self.length = length + self.offset = offset + self.box = [] + + def __str__(self): + msg = Jp2kBox.__str__(self) + for box in self.box: + boxstr = str(box) + + # Add indentation. + strs = [('\n ' + x) for x in boxstr.split('\n')] + msg += ''.join(strs) + return msg + + @staticmethod + def parse(fptr, offset, length): + """Parse codestream header box. + + Parameters + ---------- + fptr : file + Open file object. + offset : int + Start position of box in bytes. + length : int + Length of the box in bytes. + + Returns + ------- + AssociationBox instance + """ + box = CodestreamHeaderBox(length=length, offset=offset) + + # The codestream header box is a superbox, so go ahead and parse its + # child boxes. + box.box = box.parse_superbox(fptr) + + return box + + +class CompositingLayerHeaderBox(Jp2kBox): + """Container for compositing layer header box information. + + Attributes + ---------- + box_id : str + 4-character identifier for the box. + length : int + length of the box in bytes. + offset : int + offset of the box from the start of the file. + longname : str + more verbose description of the box. + box : list + List of boxes contained in this superbox. + """ + def __init__(self, length=0, offset=-1): + Jp2kBox.__init__(self, box_id='jplh', + longname='Compositing Layer Header') + self.length = length + self.offset = offset + self.box = [] + + def __str__(self): + msg = Jp2kBox.__str__(self) + for box in self.box: + boxstr = str(box) + + # Add indentation. + strs = [('\n ' + x) for x in boxstr.split('\n')] + msg += ''.join(strs) + return msg + + @staticmethod + def parse(fptr, offset, length): + """Parse compositing layer header box. + + Parameters + ---------- + fptr : file + Open file object. + offset : int + Start position of box in bytes. + length : int + Length of the box in bytes. + + Returns + ------- + AssociationBox instance + """ + box = CompositingLayerHeaderBox(length=length, offset=offset) + + # This box is a superbox, so go ahead and parse its # child boxes. + box.box = box.parse_superbox(fptr) + + return box + + +class JP2HeaderBox(Jp2kBox): + """Container for JP2 header box information. + + Attributes + ---------- + box_id : str + 4-character identifier for the box. + length : int + length of the box in bytes. + offset : int + offset of the box from the start of the file. + longname : str + more verbose description of the box. + box : list + List of boxes contained in this superbox. + """ + def __init__(self, length=0, offset=-1): + Jp2kBox.__init__(self, box_id='jp2h', longname='JP2 Header') + self.length = length + self.offset = offset + self.box = [] + + def __str__(self): + msg = Jp2kBox.__str__(self) + for box in self.box: + boxstr = str(box) + + # Add indentation. + strs = [('\n ' + x) for x in boxstr.split('\n')] + msg += ''.join(strs) + return msg + + def write(self, fptr): + """Write a JP2 Header box to file. + """ + # Write the contained boxes, then come back and write the length. + orig_pos = fptr.tell() + fptr.write(struct.pack('>I', 0)) + fptr.write('jp2h'.encode()) + for box in self.box: + box.write(fptr) + + end_pos = fptr.tell() + fptr.seek(orig_pos) + fptr.write(struct.pack('>I', end_pos - orig_pos)) + fptr.seek(end_pos) + + @staticmethod + def parse(fptr, offset, length): + """Parse JPEG 2000 header box. + + Parameters + ---------- + fptr : file + Open file object. + offset : int + Start position of box in bytes. + length : int + Length of the box in bytes. + + Returns + ------- + JP2HeaderBox instance + """ + box = JP2HeaderBox(length=length, offset=offset) + + # The JP2 header box is a superbox, so go ahead and parse its child + # boxes. + box.box = box.parse_superbox(fptr) + + return box + + class ComponentMappingBox(Jp2kBox): """Container for channel identification information. @@ -2509,11 +2698,13 @@ _BOX_WITH_ID = { 'cdef': ChannelDefinitionBox, 'cmap': ComponentMappingBox, 'colr': ColourSpecificationBox, - 'jP ': JPEG2000SignatureBox, 'ftyp': FileTypeBox, 'ihdr': ImageHeaderBox, - 'jp2h': JP2HeaderBox, + 'jP ': JPEG2000SignatureBox, + 'jpch': CodestreamHeaderBox, + 'jplh': CompositingLayerHeaderBox, 'jp2c': ContiguousCodestreamBox, + 'jp2h': JP2HeaderBox, 'lbl ': LabelBox, 'pclr': PaletteBox, 'res ': ResolutionBox, diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 178d4a0..8fdd3c3 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -19,6 +19,11 @@ from glymur.jp2box import * from glymur.core import COLOR, OPACITY from glymur.core import RED, GREEN, BLUE, GREY, WHOLE_IMAGE +try: + format_corpus_data_root = os.environ['FORMAT_CORPUS_DATA_ROOT'] +except KeyError: + format_corpus_data_root = None + # Doc tests should be run as well. def load_tests(loader, tests, ignore): @@ -673,5 +678,39 @@ class TestJp2Boxes(unittest.TestCase): with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) + +class TestJpxBoxes(unittest.TestCase): + + def setUp(self): + pass + + def tearDown(self): + pass + + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") + def test_codestream_header(self): + # Should recognize codestream header box. + jfile = os.path.join(format_corpus_data_root, + 'jp2k-formats/balloon.jpf') + jpx = Jp2k(jfile) + + # This superbox just happens to be empty. + self.assertEqual(jpx.box[4].box_id, 'jpch') + self.assertEqual(len(jpx.box[4].box), 0) + + @unittest.skipIf(format_corpus_data_root is None, + "FORMAT_CORPUS_DATA_ROOT environment variable not set") + def test_compositing_layer_header(self): + # Should recognize compositing layer header box. + jfile = os.path.join(format_corpus_data_root, + 'jp2k-formats/balloon.jpf') + jpx = Jp2k(jfile) + + # This superbox just happens to be empty. + self.assertEqual(jpx.box[5].box_id, 'jplh') + self.assertEqual(len(jpx.box[5].box), 0) + + if __name__ == "__main__": unittest.main() From 57ad32dac153931f3a9211f99514b92fcd070d9d Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 17:33:23 -0400 Subject: [PATCH 26/30] Split into two boxes, enables more tests where openjp2 not present. Closes #85 --- glymur/test/test_jp2box.py | 71 ++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 8fdd3c3..a2aa9d6 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -472,7 +472,7 @@ class TestColourSpecificationBox(unittest.TestCase): @unittest.skipIf(glymur.lib.openjp2.OPENJP2 is None, "Missing openjp2 library.") -class TestJp2Boxes(unittest.TestCase): +class TestWrap(unittest.TestCase): def setUp(self): self.j2kfile = glymur.data.goodstuff() @@ -481,39 +481,6 @@ class TestJp2Boxes(unittest.TestCase): def tearDown(self): pass - def test_default_JPEG2000SignatureBox(self): - # Should be able to instantiate a JPEG2000SignatureBox - b = glymur.jp2box.JPEG2000SignatureBox() - self.assertEqual(b.signature, (13, 10, 135, 10)) - - def test_default_FileTypeBox(self): - # Should be able to instantiate a FileTypeBox - b = glymur.jp2box.FileTypeBox() - self.assertEqual(b.brand, 'jp2 ') - self.assertEqual(b.minor_version, 0) - self.assertEqual(b.compatibility_list, ['jp2 ']) - - def test_default_ImageHeaderBox(self): - # Should be able to instantiate an image header box. - b = glymur.jp2box.ImageHeaderBox(height=512, width=256, - num_components=3) - self.assertEqual(b.height, 512) - self.assertEqual(b.width, 256) - self.assertEqual(b.num_components, 3) - self.assertEqual(b.bits_per_component, 8) - self.assertFalse(b.signed) - self.assertFalse(b.colorspace_unknown) - - def test_default_JP2HeaderBox(self): - b1 = JP2HeaderBox() - b1.box = [ImageHeaderBox(height=512, width=256), - ColourSpecificationBox(colorspace=glymur.core.GREYSCALE)] - - def test_default_ContiguousCodestreamBox(self): - b = ContiguousCodestreamBox() - self.assertEqual(b.box_id, 'jp2c') - self.assertIsNone(b.main_header) - def verify_wrapped_raw(self, jp2file): # Shared method by at least two tests. jp2 = Jp2k(jp2file) @@ -679,6 +646,42 @@ class TestJp2Boxes(unittest.TestCase): j2k.wrap(tfile.name, boxes=boxes) +class TestJp2Boxes(unittest.TestCase): + + def test_default_JPEG2000SignatureBox(self): + # Should be able to instantiate a JPEG2000SignatureBox + b = glymur.jp2box.JPEG2000SignatureBox() + self.assertEqual(b.signature, (13, 10, 135, 10)) + + def test_default_FileTypeBox(self): + # Should be able to instantiate a FileTypeBox + b = glymur.jp2box.FileTypeBox() + self.assertEqual(b.brand, 'jp2 ') + self.assertEqual(b.minor_version, 0) + self.assertEqual(b.compatibility_list, ['jp2 ']) + + def test_default_ImageHeaderBox(self): + # Should be able to instantiate an image header box. + b = glymur.jp2box.ImageHeaderBox(height=512, width=256, + num_components=3) + self.assertEqual(b.height, 512) + self.assertEqual(b.width, 256) + self.assertEqual(b.num_components, 3) + self.assertEqual(b.bits_per_component, 8) + self.assertFalse(b.signed) + self.assertFalse(b.colorspace_unknown) + + def test_default_JP2HeaderBox(self): + b1 = JP2HeaderBox() + b1.box = [ImageHeaderBox(height=512, width=256), + ColourSpecificationBox(colorspace=glymur.core.GREYSCALE)] + + def test_default_ContiguousCodestreamBox(self): + b = ContiguousCodestreamBox() + self.assertEqual(b.box_id, 'jp2c') + self.assertIsNone(b.main_header) + + class TestJpxBoxes(unittest.TestCase): def setUp(self): From 6b1eae5c4693122aba364f2fc4a118985eaa7d1a Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 17:41:10 -0400 Subject: [PATCH 27/30] Removed warning when configuration file does not exist. #86 --- glymur/lib/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/glymur/lib/config.py b/glymur/lib/config.py index 7e9e2a9..c29579c 100644 --- a/glymur/lib/config.py +++ b/glymur/lib/config.py @@ -35,9 +35,6 @@ def glymurrc_fname(): fname = os.path.join(confdir, 'glymurrc') if os.path.exists(fname): return fname - else: - msg = "Configuration directory '{0}' does not exist.".format(fname) - warnings.warn(msg) # didn't find a configuration file. return None From dcf529d25b3b03341052aa47bcd07d8e5c99531a Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 18:34:30 -0400 Subject: [PATCH 28/30] Added warning if neither library is found. #86 --- glymur/lib/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/glymur/lib/config.py b/glymur/lib/config.py index c29579c..d814b89 100644 --- a/glymur/lib/config.py +++ b/glymur/lib/config.py @@ -124,6 +124,9 @@ def glymur_config(): libs = read_config_file() libopenjp2_handle = load_openjp2(libs['openjp2']) libopenjpeg_handle = load_openjpeg(libs['openjpeg']) + if libopenjp2_handle is None and libopenjpeg_handle is None: + msg = 'Neither openjp2 nor openjpeg could not be loaded.' + warnings.warn(msg, UserWarning) return libopenjp2_handle, libopenjpeg_handle From ece145699e73e98a0de91ac4ed3116566b969660 Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 19:01:36 -0400 Subject: [PATCH 29/30] Removed warning if libraries not found; we error out first. #86 --- glymur/lib/config.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/glymur/lib/config.py b/glymur/lib/config.py index d814b89..c29579c 100644 --- a/glymur/lib/config.py +++ b/glymur/lib/config.py @@ -124,9 +124,6 @@ def glymur_config(): libs = read_config_file() libopenjp2_handle = load_openjp2(libs['openjp2']) libopenjpeg_handle = load_openjpeg(libs['openjpeg']) - if libopenjp2_handle is None and libopenjpeg_handle is None: - msg = 'Neither openjp2 nor openjpeg could not be loaded.' - warnings.warn(msg, UserWarning) return libopenjp2_handle, libopenjpeg_handle From 0a81c88c4bb4ebca9932a0bb090668cadf74a61f Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 24 Jul 2013 19:02:08 -0400 Subject: [PATCH 30/30] Removed tests for warnings if config file not found. Warnings in this situation are no longer desirable. Closes #86. --- glymur/test/test_config.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/glymur/test/test_config.py b/glymur/test/test_config.py index 09370e0..7833865 100644 --- a/glymur/test/test_config.py +++ b/glymur/test/test_config.py @@ -81,40 +81,5 @@ class TestSuite(unittest.TestCase): with self.assertWarns(UserWarning) as cw: imp.reload(glymur.lib.openjp2) - def test_missing_config_file_via_environ(self): - # Verify that we error out properly if the configuration file - # specified via environment variable is not found. - with tempfile.TemporaryDirectory() as tdir: - with patch.dict('os.environ', {'XDG_CONFIG_HOME': tdir}): - # Misconfigured new configuration file should - # be rejected. - with self.assertWarns(UserWarning) as cw: - imp.reload(glymur.lib.openjp2) - - def test_home_dir_missing_config_dir(self): - # Verify no exception is raised if $HOME is missing .config directory. - with tempfile.TemporaryDirectory() as tdir: - with patch.dict('os.environ', {'HOME': tdir}): - # Misconfigured new configuration file should - # be rejected. - with self.assertWarns(UserWarning) as cw: - imp.reload(glymur.lib.openjp2) - - def test_home_dir_missing_glymur_rc_dir(self): - # Should warn but not error if $HOME/.config but no glymurrc dir. - with tempfile.TemporaryDirectory() as tdir: - # We need the subdirectory to be specifically named as ".config" - # in order for this test to work. A specifically-named temporary - # directory does not seem to be possible, so try to symlink it. - # Supposedly the symlink gets cleaned up with tdir gets cleaned up. - with tempfile.TemporaryDirectory(suffix=".config", dir=tdir) \ - as tdir_config: - os.symlink(tdir_config, os.path.join(tdir, '.config')) - with patch.dict('os.environ', {'HOME': tdir}): - # Misconfigured new configuration file should - # be rejected. - with self.assertWarns(UserWarning) as cw: - imp.reload(glymur.lib.openjp2) - if __name__ == "__main__": unittest.main()