From 0e302a571000a465d34f7d3632e1c1de03bcbdae Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 20 Oct 2013 11:57:51 -0400 Subject: [PATCH 01/83] Adding XMP UUID write support. Still needs tests. #104 --- glymur/jp2box.py | 19 +++++++++++++++++++ glymur/jp2k.py | 9 ++++++--- glymur/test/test_jp2box.py | 16 ++++++++-------- setup.py | 2 +- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index bf88e4d..6f628df 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2122,6 +2122,7 @@ class UUIDBox(Jp2kBox): text = raw_data.decode('utf-8') elt = ET.fromstring(text) self.data = ET.ElementTree(elt) + self._type = 'XMP' elif the_uuid.bytes == b'JpgTiffExif->JP2': exif_obj = Exif(raw_data) ifds = OrderedDict() @@ -2130,8 +2131,16 @@ class UUIDBox(Jp2kBox): ifds['GPSInfo'] = exif_obj.exif_gpsinfo ifds['Iop'] = exif_obj.exif_iop self.data = ifds + self._type = 'Exif' else: self.data = raw_data + self._type = 'unknown' + + if length == 0: + # Need to compute the length. + # The length is 8 (L and T fields) + 16 (length of UUID identifier) + # + length of uuid data. + length = 24 + len(self.data) self.length = length self.offset = offset @@ -2164,6 +2173,16 @@ class UUIDBox(Jp2kBox): return msg + def write(self, fptr): + """Write a UUID box box to file. + """ + if self._type != 'XMP': + msg = "Only XMP UUID boxes can currently be written." + raise NotImplementedError(msg) + read_buffer = struct.pack('>I4s', self.length, 'uuid') + fptr.write(read_buffer) + fptr.write(self.data) + @staticmethod def parse(fptr, offset, length): """Parse UUID box. diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 1bd36a4..45eaf39 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -514,14 +514,17 @@ class Jp2k(Jp2kBox): Parameters ---------- box : Jp2Box - Instance of a JP2 box. Currently only XML boxes are allowed. + Instance of a JP2 box. Only UUID and XML boxes can currently be + appended. """ if self._codec_format == opj2.CODEC_J2K: msg = "Only JP2 files can currently have boxes appended to them." raise IOError(msg) - if box.box_id != 'xml ': - raise IOError("Only XML boxes can currently be appended.") + if not ((box.box_id == 'xml ') or + (box.box_id == 'uuid' and box._type == 'XMP')): + msg = "Only XML boxes and XMP UUID boxes can currently be appended." + raise IOError(msg) # Check the last box. If the length field is zero, then rewrite # the length field to reflect the true length of the box. diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index ef62460..ff61bc5 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -429,14 +429,14 @@ class TestAppend(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".j2k") as tfile: shutil.copyfile(self.j2kfile, tfile.name) - jp2 = Jp2k(tfile.name) + j2k = Jp2k(tfile.name) - # Make a UUID box. - uuid_instance = uuid.UUID('00000000-0000-0000-0000-000000000000') - data = b'0123456789' - uuidbox = glymur.jp2box.UUIDBox(uuid_instance, data) + # Make an XML box. XML boxes should always be appendable to jp2 + # files. + the_xml = ET.fromstring('0') + xmlbox = glymur.jp2box.XMLBox(xml=the_xml) with self.assertRaises(IOError): - jp2.append(uuidbox) + j2k.append(xmlbox) def test_length_field_is_zero(self): """L=0 (length field in box header) is handled. @@ -473,14 +473,14 @@ class TestAppend(unittest.TestCase): self.assertEqual(ET.tostring(jp2.box[-1].xml.getroot()), b'0') - def test_only_xml_allowed_to_append(self): + def test_append_allowable_boxes(self): """Only XML boxes are allowed to be appended.""" with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: shutil.copyfile(self.jp2file, tfile.name) jp2 = Jp2k(tfile.name) - # Make a UUID box. + # Make a UUID box. Only XMP UUID boxes can currently be appended. uuid_instance = uuid.UUID('00000000-0000-0000-0000-000000000000') data = b'0123456789' uuidbox = glymur.jp2box.UUIDBox(uuid_instance, data) diff --git a/setup.py b/setup.py index 89e69f7..b780e4d 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ kwargs['classifiers'] = clssfrs # Get the version string. Cannot do this by importing glymur! version_file = os.path.join('glymur', 'version.py') -with open('glymur/version.py', 'rt') as fptr: +with open(version_file, 'rt') as fptr: contents = fptr.read() match = re.search('version\s*=\s*"(?P\d*.\d*.\d*.*)"\n', contents) kwargs['version'] = match.group('version') From 08aaa25fdd03dcad140c1bd6b82b31840d633961 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 24 Oct 2013 19:05:11 -0400 Subject: [PATCH 02/83] Changed nemo.jp2 to have a single XMP UUID. #104 --- glymur/data/nemo.jp2 | Bin 1135423 -> 1135519 bytes glymur/jp2box.py | 16 ++--- glymur/jp2k.py | 2 +- glymur/test/fixtures.py | 87 ++++++++++++++++++++++++ glymur/test/test_jp2box.py | 4 +- glymur/test/test_jp2k.py | 78 ++++++--------------- glymur/test/test_printing.py | 127 ++++++++++++++--------------------- 7 files changed, 169 insertions(+), 145 deletions(-) diff --git a/glymur/data/nemo.jp2 b/glymur/data/nemo.jp2 index 55d199cfe6ebc1630bd87a238c6f4cb4b7333d9d..838583d2f3e22a49d877299178a764436b808e74 100644 GIT binary patch delta 2966 zcma)8&2QsG6c^3%Rmzt*f<(3|C$_OY9w)KYT7t9q!gl{X$|VkgdO4^Oh+`@J{sYu>!Se?R~AkMp0e_YS^Z{=4_t z^1r>u-=BnuYAu2&AgR+_1b&!Uo_l&YOVh|wlxQCNEf!A|&sB&|0urW)qO~;Tu)u=$ z>7nDX7sLWhZloka%YPc5W)i7OvtiQ8by_Y9l$1^;U{ftA;(8=m&`qRKkBnw4Wyy?1 ziuQW8`^}Tsn^?oMi{&qk#~J)L zn~)=9^e~`BgAEb5dd>Rnw1EbgY_V4O_Tl8G1=kU_kFg;m4kl z#Eof`QWo-)PnN$mK9N4?)?M#KFCMUjasUmdw#H4>@C!$D>vl*I=FfA39i$ZaGFL+H zYN2<-Y;8ee*U)c6n%YRXvB8z-E9airUyXYLmH(|}%p*P*Hv7KgQW6)SoSmaG9^qx+ zOvw%P(wU76vpBEBl=gDoQxZ-AsbQln%$2zk`Xx=10f|RW5cwo8w5a`v{2>0sZ{}0G zE6M|3TpJ3*i}mV!p}n#zT-rVz+Dx|wymR3m#ninFQxZpU2A3+YUlH{-Vg-)z{gM#SxI6;Ns9a&$e42=Rt+9m46$??MNLU@@<_ ze(}_lrxCX5?B$`I!<+k@WN}_a6`>+&f-ShxsSDppW)WRHCk1?6jtz-a>9HVUAzv$c zES%8kJQlM~ILf`w{oJ^N)p*wTZ3iF5Upan>hgiyd@mI}D;a^tk;Hvgd!Drr7Hbzg7 z+A`YDL=!^!rAS81nJkbXd&}vjhCXb!Z?F|8Kv-0Rhq>V>%@0bo41vKm9+;k zb4ijwpIni2#ytDk`S|8~_>ENC0)Vzi#%d@u;12VB=z7)jM&6wHPDH$lQ>7iJ3feMS z*eqs?=dM<)mk##6{}9Y!NhlXi7Ig3c@rC(L$M>{*npT0E70%CW=X)JLv>G!XTd)o0 zC&4;s@AG`gUD^Ke=-{WLgV%uefEqxLfL;fB1L#elw}9RT+6Q_Eh~wV{dJpJ*pb!2! JI{5J8{{NWigc<+< delta 2973 zcmcJN&rTCj6vpq&v=pX=q7WBWlVKu}kkFZy63e7CF>NhHE&rO}w%C?5wzO6t)rFHN z3B*;A#I3CK5nQ-nqHV4yH`lCSqW=bn3h=ic<|_ssFPOlMK&-*0bM zOAF17LbX!Kw5ye={3rPFeRv&ao$la6b17RZzO0s8WF9e1|6l#&;lA1C87=MjjB(Ux zR24cr8AWOot_rEsxmQh^lf&`sr-^kvZG%0YU`kJ6=M(%%|yL zd{H$rt*KE>os}XnE%r#8nLamHM12YMJT79s-pXto?i7zro%r#~5rG!swP-1Qv^2Zi)rY%bDgM1G)4=qgDi{j7)JhApP5H!#3O7~*gC`3we0FtJVwal)l@ErTs~7+TU>c+c`Zyy z$UyC2d0hDW^Y~zAEH5V7TD#t?m$!;iyI$LTqqXPcVyUrS*5GMcktL&K>(!k6;`x%4 zYP8ByEF24~Q~2BNMCGI?C0eD5wvv8w_=SrfCjDbWqJ)q2t2Yl{b9eN1v8hDDsv^@p zAhlGhT-<833XMi>PTr`!t#7wvg*H=Ix1^X*PNu_#t55Ilv`dOJ#K!L##mcusY%PCkjT|Tr)krTpZ#nN=L=ivk o2KIIX9K=HcI4s', self.length, 'uuid') + serialized_buffer = b'' + serialized_buffer += ET.tostring(self.data.getroot(), encoding='utf-8') + serialized_buffer += b'' + if self.length == 0: + self.length = 24 + len(serialized_buffer) + read_buffer = struct.pack('>I4s', self.length, b'uuid') fptr.write(read_buffer) - fptr.write(self.data) + fptr.write(self.uuid.bytes) + fptr.write(serialized_buffer) @staticmethod def parse(fptr, offset, length): diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 45eaf39..42eaab5 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1013,7 +1013,7 @@ class Jp2k(Jp2kBox): >>> jp2 = glymur.Jp2k(jfile) >>> codestream = jp2.get_codestream() >>> print(codestream.segment[1]) - SIZ marker segment @ (3137, 47) + SIZ marker segment @ (3233, 47) Profile: 2 Reference Grid Height, Width: (1456 x 2592) Vertical, Horizontal Reference Grid Offset: (0 x 0) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index b872f1d..68c0706 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -167,3 +167,90 @@ def read_pgx_header(pgx_file): header = header.rstrip() return header, pos + +nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + """ diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index ff61bc5..2f6241c 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -419,7 +419,7 @@ class TestAppend(unittest.TestCase): # The sequence of box IDs should be the same as before, but with an # xml box at the end. box_ids = [box.box_id for box in jp2.box] - expected = ['jP ', 'ftyp', 'jp2h', 'uuid', 'uuid', 'jp2c', 'xml '] + expected = ['jP ', 'ftyp', 'jp2h', 'uuid', 'jp2c', 'xml '] self.assertEqual(box_ids, expected) self.assertEqual(ET.tostring(jp2.box[-1].xml.getroot()), b'0') @@ -468,7 +468,7 @@ class TestAppend(unittest.TestCase): # The sequence of box IDs should be the same as before, but with an # xml box at the end. box_ids = [box.box_id for box in jp2.box] - expected = ['jP ', 'ftyp', 'jp2h', 'uuid', 'uuid', 'jp2c', 'xml '] + expected = ['jP ', 'ftyp', 'jp2h', 'uuid', 'jp2c', 'xml '] self.assertEqual(box_ids, expected) self.assertEqual(ET.tostring(jp2.box[-1].xml.getroot()), b'0') diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index 636ea9a..c9e750a 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -100,7 +100,7 @@ class TestJp2k(unittest.TestCase): jp2k = Jp2k(self.jp2file) # top-level boxes - self.assertEqual(len(jp2k.box), 6) + self.assertEqual(len(jp2k.box), 5) self.assertEqual(jp2k.box[0].box_id, 'jP ') self.assertEqual(jp2k.box[0].offset, 0) @@ -119,15 +119,11 @@ class TestJp2k(unittest.TestCase): self.assertEqual(jp2k.box[3].box_id, 'uuid') self.assertEqual(jp2k.box[3].offset, 77) - self.assertEqual(jp2k.box[3].length, 638) + self.assertEqual(jp2k.box[3].length, 3146) - self.assertEqual(jp2k.box[4].box_id, 'uuid') - self.assertEqual(jp2k.box[4].offset, 715) - self.assertEqual(jp2k.box[4].length, 2412) - - self.assertEqual(jp2k.box[5].box_id, 'jp2c') - self.assertEqual(jp2k.box[5].offset, 3127) - self.assertEqual(jp2k.box[5].length, 1132296) + self.assertEqual(jp2k.box[4].box_id, 'jp2c') + self.assertEqual(jp2k.box[4].offset, 3223) + self.assertEqual(jp2k.box[4].length, 1132296) # jp2h super box self.assertEqual(len(jp2k.box[2].box), 2) @@ -169,7 +165,7 @@ class TestJp2k(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile: with open(self.jp2file, 'rb') as ifile: # Everything up until the jp2c box. - write_buffer = ifile.read(3127) + write_buffer = ifile.read(3223) tfile.write(write_buffer) # The L field must be 1 in order to signal the presence of the @@ -190,9 +186,9 @@ class TestJp2k(unittest.TestCase): jp2k = Jp2k(tfile.name) - self.assertEqual(jp2k.box[5].box_id, 'jp2c') - self.assertEqual(jp2k.box[5].offset, 3127) - self.assertEqual(jp2k.box[5].length, 1133427 + 8) + self.assertEqual(jp2k.box[4].box_id, 'jp2c') + self.assertEqual(jp2k.box[4].offset, 3223) + self.assertEqual(jp2k.box[4].length, 1133427 + 8) @unittest.skipIf(os.name == "nt", "NamedTemporaryFile issue on windows") def test_length_field_is_zero(self): @@ -357,45 +353,12 @@ class TestJp2k(unittest.TestCase): def test_xmp_attribute(self): """Verify the XMP packet in the shipping example file can be read.""" j = Jp2k(self.jp2file) - xmp = j.box[4].data + xmp = j.box[3].data ns0 = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' - ns1 = '{http://ns.adobe.com/xap/1.0/}' - name = '{0}RDF/{0}Description'.format(ns0) + ns2 = '{http://ns.adobe.com/xap/1.0/}' + name = '{0}RDF/{0}Description/{1}CreatorTool'.format(ns0, ns2) elt = xmp.find(name) - attr_value = elt.attrib['{0}CreatorTool'.format(ns1)] - self.assertEqual(attr_value, 'glymur') - - @unittest.skipIf(os.name == "nt", "NamedTemporaryFile issue on windows") - def test_unrecognized_exif_tag(self): - """An unrecognized exif tag should be handled gracefully.""" - with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile: - shutil.copyfile(self.jp2file, tfile.name) - - # The Exif UUID starts at byte 77. There are 8 bytes for the L and - # T fields, then 16 bytes for the UUID identifier, then 6 exif - # header bytes, then 8 bytes for the TIFF header, then 2 bytes - # the the Image IFD number of tags, where we finally find the first - # tag, "Make" (271). We'll corrupt it by changing it into 171, - # which does not correspond to any known Exif Image tag. - with open(tfile.name, 'r+b') as fptr: - fptr.seek(117) - write_buffer = struct.pack('', - ' ', - ' ', - ' ', - ' '] - expected = '\n'.join(lst) + expected = nemo_xmp_box self.assertEqual(actual, expected) def test_codestream(self): @@ -657,8 +645,8 @@ class TestPrinting(unittest.TestCase): print(j.get_codestream()) actual = fake_out.getvalue().strip() lst = ['Codestream:', - ' SOC marker segment @ (3135, 0)', - ' SIZ marker segment @ (3137, 47)', + ' SOC marker segment @ (3231, 0)', + ' SIZ marker segment @ (3233, 47)', ' Profile: 2', ' Reference Grid Height, Width: (1456 x 2592)', ' Vertical, Horizontal Reference Grid Offset: (0 x 0)', @@ -668,7 +656,7 @@ class TestPrinting(unittest.TestCase): ' Signed: (False, False, False)', ' Vertical, Horizontal Subsampling: ' + '((1, 1), (1, 1), (1, 1))', - ' COD marker segment @ (3186, 12)', + ' COD marker segment @ (3282, 12)', ' Coding style:', ' Entropy coder, without partitions', ' SOP marker segments: False', @@ -690,11 +678,11 @@ class TestPrinting(unittest.TestCase): ' Vertically stripe causal context: False', ' Predictable termination: False', ' Segmentation symbols: False', - ' QCD marker segment @ (3200, 7)', + ' QCD marker segment @ (3296, 7)', ' Quantization style: no quantization, ' + '2 guard bits', ' Step size: [(0, 8), (0, 9), (0, 9), (0, 10)]', - ' CME marker segment @ (3209, 37)', + ' CME marker segment @ (3305, 37)', ' "Created by OpenJPEG version 2.0.0"'] expected = '\n'.join(lst) self.assertEqual(actual, expected) @@ -1046,59 +1034,42 @@ class TestPrinting(unittest.TestCase): "Ordered dicts not printing well in 2.7") def test_exif_uuid(self): """Verify printing of exif information""" - j = glymur.Jp2k(self.jp2file) + with tempfile.NamedTemporaryFile(suffix='.jp2', mode='wb') as tfile: - with patch('sys.stdout', new=StringIO()) as fake_out: - print(j.box[3]) - actual = fake_out.getvalue().strip() + with open(self.jp2file, 'rb') as ifptr: + tfile.write(ifptr.read()) - lines = ["UUID Box (uuid) @ (77, 638)", + # Write L, T, UUID identifier. + tfile.write(struct.pack('>I4s', 76, b'uuid')) + tfile.write(b'JpgTiffExif->JP2') + + tfile.write(b'Exif\x00\x00') + xbuffer = struct.pack(' Date: Thu, 24 Oct 2013 20:38:29 -0400 Subject: [PATCH 03/83] Refactored Exif uuid code to separate subpackage. #104 --- CHANGES.txt | 4 + glymur/_uuid_io/Exif.py | 520 ++++++++++++++++++++++++++++++++ glymur/_uuid_io/__init__.py | 1 + glymur/jp2box.py | 509 +------------------------------ glymur/test/test_jp2box_uuid.py | 78 +++++ 5 files changed, 606 insertions(+), 506 deletions(-) create mode 100644 glymur/_uuid_io/Exif.py create mode 100644 glymur/_uuid_io/__init__.py create mode 100644 glymur/test/test_jp2box_uuid.py diff --git a/CHANGES.txt b/CHANGES.txt index 40846e4..65270bf 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,7 @@ +Oct 24, 2013 - v0.6.0 Palette box palette changed to 2D numpy array. Removed + nemo Exif and simple XMP UUIDs in favor of larger XMP UUID. Refactored + Exif UUID code into _uuid_io sub package. + Oct 13, 2013 - v0.5.6 Fixed handling of non-ascii chars in XML boxes. Fixed some docstring errors in jp2box module. diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py new file mode 100644 index 0000000..f70a090 --- /dev/null +++ b/glymur/_uuid_io/Exif.py @@ -0,0 +1,520 @@ +# -*- coding: utf-8 -*- +""" +Handlers for various UUID types. +""" +import struct +import sys +import warnings + +if sys.hexversion < 0x02070000: + # pylint: disable=F0401,E0611 + from ordereddict import OrderedDict +else: + from collections import OrderedDict + +class _Exif(object): + """ + Attributes + ---------- + read_buffer : bytes + Raw byte stream consisting of the UUID data. + endian : str + Either '<' for big-endian, or '>' for little-endian. + """ + + def __init__(self, read_buffer): + """Interpret raw buffer consisting of Exif IFD. + """ + self.exif_image = None + self.exif_photo = None + self.exif_gpsinfo = None + self.exif_iop = None + + self.read_buffer = read_buffer + + # Ignore the first six bytes. + # Next 8 should be (73, 73, 42, 8) + data = struct.unpack('' for little-endian. + num_tags : int + Number of tags in the IFD. + raw_ifd : dictionary + Maps tag number to "mildly-interpreted" tag value. + processed_ifd : dictionary + Maps tag name to "mildly-interpreted" tag value. + """ + datatype2fmt = {1: ('B', 1), + 2: ('B', 1), + 3: ('H', 2), + 4: ('I', 4), + 5: ('II', 8), + 7: ('B', 1), + 9: ('i', 4), + 10: ('ii', 8)} + + def __init__(self, endian, read_buffer, offset): + self.endian = endian + self.read_buffer = read_buffer + self.processed_ifd = OrderedDict() + + self.num_tags, = struct.unpack(endian + 'H', + read_buffer[offset:offset + 2]) + + fmt = self.endian + 'HHII' * self.num_tags + ifd_buffer = read_buffer[offset + 2:offset + 2 + self.num_tags * 12] + data = struct.unpack(fmt, ifd_buffer) + self.raw_ifd = OrderedDict() + for j, tag in enumerate(data[0::4]): + # The offset to the tag offset/payload is the offset to the IFD + # plus 2 bytes for the number of tags plus 12 bytes for each + # tag entry plus 8 bytes to the offset/payload itself. + toffp = read_buffer[offset + 10 + j * 12:offset + 10 + j * 12 + 4] + tag_data = self.parse_tag(data[j * 4 + 1], + data[j * 4 + 2], + toffp) + self.raw_ifd[tag] = tag_data + + def parse_tag(self, dtype, count, offset_buf): + """Interpret an Exif image tag data payload. + """ + fmt = self.datatype2fmt[dtype][0] * count + payload_size = self.datatype2fmt[dtype][1] * count + + if payload_size <= 4: + # Interpret the payload from the 4 bytes in the tag entry. + target_buffer = offset_buf[:payload_size] + else: + # Interpret the payload at the offset specified by the 4 bytes in + # the tag entry. + offset, = struct.unpack(self.endian + 'I', offset_buf) + target_buffer = self.read_buffer[offset:offset + payload_size] + + if dtype == 2: + # ASCII + if sys.hexversion < 0x03000000: + payload = target_buffer.rstrip('\x00') + else: + payload = target_buffer.decode('utf-8').rstrip('\x00') + + else: + payload = struct.unpack(self.endian + fmt, target_buffer) + if dtype == 5 or dtype == 10: + # Rational or Signed Rational. Construct the list of values. + rational_payload = [] + for j in range(count): + value = float(payload[j * 2]) / float(payload[j * 2 + 1]) + rational_payload.append(value) + payload = rational_payload + if count == 1: + # If just a single value, then return a scalar instead of a + # tuple. + payload = payload[0] + + return payload + + def post_process(self, tagnum2name): + """Map the tag name instead of tag number to the tag value. + """ + for tag, value in self.raw_ifd.items(): + try: + tag_name = tagnum2name[tag] + except KeyError: + # Ok, we don't recognize this tag. Just use the numeric id. + msg = 'Unrecognized Exif tag "{0}".'.format(tag) + warnings.warn(msg, UserWarning) + tag_name = tag + self.processed_ifd[tag_name] = value + + +class _ExifImageIfd(_Ifd): + """ + Attributes + ---------- + tagnum2name : dict + Maps Exif image tag numbers to the tag names. + ifd : dict + Maps tag names to tag values. + """ + tagnum2name = {11: 'ProcessingSoftware', + 254: 'NewSubfileType', + 255: 'SubfileType', + 256: 'ImageWidth', + 257: 'ImageLength', + 258: 'BitsPerSample', + 259: 'Compression', + 262: 'PhotometricInterpretation', + 263: 'Threshholding', + 264: 'CellWidth', + 265: 'CellLength', + 266: 'FillOrder', + 269: 'DocumentName', + 270: 'ImageDescription', + 271: 'Make', + 272: 'Model', + 273: 'StripOffsets', + 274: 'Orientation', + 277: 'SamplesPerPixel', + 278: 'RowsPerStrip', + 279: 'StripByteCounts', + 282: 'XResolution', + 283: 'YResolution', + 284: 'PlanarConfiguration', + 290: 'GrayResponseUnit', + 291: 'GrayResponseCurve', + 292: 'T4Options', + 293: 'T6Options', + 296: 'ResolutionUnit', + 301: 'TransferFunction', + 305: 'Software', + 306: 'DateTime', + 315: 'Artist', + 316: 'HostComputer', + 317: 'Predictor', + 318: 'WhitePoint', + 319: 'PrimaryChromaticities', + 320: 'ColorMap', + 321: 'HalftoneHints', + 322: 'TileWidth', + 323: 'TileLength', + 324: 'TileOffsets', + 325: 'TileByteCounts', + 330: 'SubIFDs', + 332: 'InkSet', + 333: 'InkNames', + 334: 'NumberOfInks', + 336: 'DotRange', + 337: 'TargetPrinter', + 338: 'ExtraSamples', + 339: 'SampleFormat', + 340: 'SMinSampleValue', + 341: 'SMaxSampleValue', + 342: 'TransferRange', + 343: 'ClipPath', + 344: 'XClipPathUnits', + 345: 'YClipPathUnits', + 346: 'Indexed', + 347: 'JPEGTables', + 351: 'OPIProxy', + 512: 'JPEGProc', + 513: 'JPEGInterchangeFormat', + 514: 'JPEGInterchangeFormatLength', + 515: 'JPEGRestartInterval', + 517: 'JPEGLosslessPredictors', + 518: 'JPEGPointTransforms', + 519: 'JPEGQTables', + 520: 'JPEGDCTables', + 521: 'JPEGACTables', + 529: 'YCbCrCoefficients', + 530: 'YCbCrSubSampling', + 531: 'YCbCrPositioning', + 532: 'ReferenceBlackWhite', + 700: 'XMLPacket', + 18246: 'Rating', + 18249: 'RatingPercent', + 32781: 'ImageID', + 33421: 'CFARepeatPatternDim', + 33422: 'CFAPattern', + 33423: 'BatteryLevel', + 33432: 'Copyright', + 33434: 'ExposureTime', + 33437: 'FNumber', + 33723: 'IPTCNAA', + 34377: 'ImageResources', + 34665: 'ExifTag', + 34675: 'InterColorProfile', + 34850: 'ExposureProgram', + 34852: 'SpectralSensitivity', + 34853: 'GPSTag', + 34855: 'ISOSpeedRatings', + 34856: 'OECF', + 34857: 'Interlace', + 34858: 'TimeZoneOffset', + 34859: 'SelfTimerMode', + 36867: 'DateTimeOriginal', + 37122: 'CompressedBitsPerPixel', + 37377: 'ShutterSpeedValue', + 37378: 'ApertureValue', + 37379: 'BrightnessValue', + 37380: 'ExposureBiasValue', + 37381: 'MaxApertureValue', + 37382: 'SubjectDistance', + 37383: 'MeteringMode', + 37384: 'LightSource', + 37385: 'Flash', + 37386: 'FocalLength', + 37387: 'FlashEnergy', + 37388: 'SpatialFrequencyResponse', + 37389: 'Noise', + 37390: 'FocalPlaneXResolution', + 37391: 'FocalPlaneYResolution', + 37392: 'FocalPlaneResolutionUnit', + 37393: 'ImageNumber', + 37394: 'SecurityClassification', + 37395: 'ImageHistory', + 37396: 'SubjectLocation', + 37397: 'ExposureIndex', + 37398: 'TIFFEPStandardID', + 37399: 'SensingMethod', + 40091: 'XPTitle', + 40092: 'XPComment', + 40093: 'XPAuthor', + 40094: 'XPKeywords', + 40095: 'XPSubject', + 50341: 'PrintImageMatching', + 50706: 'DNGVersion', + 50707: 'DNGBackwardVersion', + 50708: 'UniqueCameraModel', + 50709: 'LocalizedCameraModel', + 50710: 'CFAPlaneColor', + 50711: 'CFALayout', + 50712: 'LinearizationTable', + 50713: 'BlackLevelRepeatDim', + 50714: 'BlackLevel', + 50715: 'BlackLevelDeltaH', + 50716: 'BlackLevelDeltaV', + 50717: 'WhiteLevel', + 50718: 'DefaultScale', + 50719: 'DefaultCropOrigin', + 50720: 'DefaultCropSize', + 50721: 'ColorMatrix1', + 50722: 'ColorMatrix2', + 50723: 'CameraCalibration1', + 50724: 'CameraCalibration2', + 50725: 'ReductionMatrix1', + 50726: 'ReductionMatrix2', + 50727: 'AnalogBalance', + 50728: 'AsShotNeutral', + 50729: 'AsShotWhiteXY', + 50730: 'BaselineExposure', + 50731: 'BaselineNoise', + 50732: 'BaselineSharpness', + 50733: 'BayerGreenSplit', + 50734: 'LinearResponseLimit', + 50735: 'CameraSerialNumber', + 50736: 'LensInfo', + 50737: 'ChromaBlurRadius', + 50738: 'AntiAliasStrength', + 50739: 'ShadowScale', + 50740: 'DNGPrivateData', + 50741: 'MakerNoteSafety', + 50778: 'CalibrationIlluminant1', + 50779: 'CalibrationIlluminant2', + 50780: 'BestQualityScale', + 50781: 'RawDataUniqueID', + 50827: 'OriginalRawFileName', + 50828: 'OriginalRawFileData', + 50829: 'ActiveArea', + 50830: 'MaskedAreas', + 50831: 'AsShotICCProfile', + 50832: 'AsShotPreProfileMatrix', + 50833: 'CurrentICCProfile', + 50834: 'CurrentPreProfileMatrix', + 50879: 'ColorimetricReference', + 50931: 'CameraCalibrationSignature', + 50932: 'ProfileCalibrationSignature', + 50934: 'AsShotProfileName', + 50935: 'NoiseReductionApplied', + 50936: 'ProfileName', + 50937: 'ProfileHueSatMapDims', + 50938: 'ProfileHueSatMapData1', + 50939: 'ProfileHueSatMapData2', + 50940: 'ProfileToneCurve', + 50941: 'ProfileEmbedPolicy', + 50942: 'ProfileCopyright', + 50964: 'ForwardMatrix1', + 50965: 'ForwardMatrix2', + 50966: 'PreviewApplicationName', + 50967: 'PreviewApplicationVersion', + 50968: 'PreviewSettingsName', + 50969: 'PreviewSettingsDigest', + 50970: 'PreviewColorSpace', + 50971: 'PreviewDateTime', + 50972: 'RawImageDigest', + 50973: 'OriginalRawFileDigest', + 50974: 'SubTileBlockSize', + 50975: 'RowInterleaveFactor', + 50981: 'ProfileLookTableDims', + 50982: 'ProfileLookTableData', + 51008: 'OpcodeList1', + 51009: 'OpcodeList2', + 51022: 'OpcodeList3', + 51041: 'NoiseProfile'} + + def __init__(self, endian, read_buffer, offset): + _Ifd.__init__(self, endian, read_buffer, offset) + self.post_process(self.tagnum2name) + + +class _ExifPhotoIfd(_Ifd): + """Represents tags found in the Exif sub ifd. + """ + tagnum2name = {33434: 'ExposureTime', + 33437: 'FNumber', + 34850: 'ExposureProgram', + 34852: 'SpectralSensitivity', + 34855: 'ISOSpeedRatings', + 34856: 'OECF', + 34864: 'SensitivityType', + 34865: 'StandardOutputSensitivity', + 34866: 'RecommendedExposureIndex', + 34867: 'ISOSpeed', + 34868: 'ISOSpeedLatitudeyyy', + 34869: 'ISOSpeedLatitudezzz', + 36864: 'ExifVersion', + 36867: 'DateTimeOriginal', + 36868: 'DateTimeDigitized', + 37121: 'ComponentsConfiguration', + 37122: 'CompressedBitsPerPixel', + 37377: 'ShutterSpeedValue', + 37378: 'ApertureValue', + 37379: 'BrightnessValue', + 37380: 'ExposureBiasValue', + 37381: 'MaxApertureValue', + 37382: 'SubjectDistance', + 37383: 'MeteringMode', + 37384: 'LightSource', + 37385: 'Flash', + 37386: 'FocalLength', + 37396: 'SubjectArea', + 37500: 'MakerNote', + 37510: 'UserComment', + 37520: 'SubSecTime', + 37521: 'SubSecTimeOriginal', + 37522: 'SubSecTimeDigitized', + 40960: 'FlashpixVersion', + 40961: 'ColorSpace', + 40962: 'PixelXDimension', + 40963: 'PixelYDimension', + 40964: 'RelatedSoundFile', + 40965: 'InteroperabilityTag', + 41483: 'FlashEnergy', + 41484: 'SpatialFrequencyResponse', + 41486: 'FocalPlaneXResolution', + 41487: 'FocalPlaneYResolution', + 41488: 'FocalPlaneResolutionUnit', + 41492: 'SubjectLocation', + 41493: 'ExposureIndex', + 41495: 'SensingMethod', + 41728: 'FileSource', + 41729: 'SceneType', + 41730: 'CFAPattern', + 41985: 'CustomRendered', + 41986: 'ExposureMode', + 41987: 'WhiteBalance', + 41988: 'DigitalZoomRatio', + 41989: 'FocalLengthIn35mmFilm', + 41990: 'SceneCaptureType', + 41991: 'GainControl', + 41992: 'Contrast', + 41993: 'Saturation', + 41994: 'Sharpness', + 41995: 'DeviceSettingDescription', + 41996: 'SubjectDistanceRange', + 42016: 'ImageUniqueID', + 42032: 'CameraOwnerName', + 42033: 'BodySerialNumber', + 42034: 'LensSpecification', + 42035: 'LensMake', + 42036: 'LensModel', + 42037: 'LensSerialNumber'} + + def __init__(self, endian, read_buffer, offset): + _Ifd.__init__(self, endian, read_buffer, offset) + self.post_process(self.tagnum2name) + + +class _ExifGPSInfoIfd(_Ifd): + """Represents information found in the GPSInfo sub IFD. + """ + tagnum2name = {0: 'GPSVersionID', + 1: 'GPSLatitudeRef', + 2: 'GPSLatitude', + 3: 'GPSLongitudeRef', + 4: 'GPSLongitude', + 5: 'GPSAltitudeRef', + 6: 'GPSAltitude', + 7: 'GPSTimeStamp', + 8: 'GPSSatellites', + 9: 'GPSStatus', + 10: 'GPSMeasureMode', + 11: 'GPSDOP', + 12: 'GPSSpeedRef', + 13: 'GPSSpeed', + 14: 'GPSTrackRef', + 15: 'GPSTrack', + 16: 'GPSImgDirectionRef', + 17: 'GPSImgDirection', + 18: 'GPSMapDatum', + 19: 'GPSDestLatitudeRef', + 20: 'GPSDestLatitude', + 21: 'GPSDestLongitudeRef', + 22: 'GPSDestLongitude', + 23: 'GPSDestBearingRef', + 24: 'GPSDestBearing', + 25: 'GPSDestDistanceRef', + 26: 'GPSDestDistance', + 27: 'GPSProcessingMethod', + 28: 'GPSAreaInformation', + 29: 'GPSDateStamp', + 30: 'GPSDifferential'} + + def __init__(self, endian, read_buffer, offset): + _Ifd.__init__(self, endian, read_buffer, offset) + self.post_process(self.tagnum2name) + + +class _ExifInteroperabilityIfd(_Ifd): + """Represents tags found in the Interoperability sub IFD. + """ + tagnum2name = {1: 'InteroperabilityIndex', + 2: 'InteroperabilityVersion', + 4096: 'RelatedImageFileFormat', + 4097: 'RelatedImageWidth', + 4098: 'RelatedImageLength'} + + def __init__(self, endian, read_buffer, offset): + _Ifd.__init__(self, endian, read_buffer, offset) + self.post_process(self.tagnum2name) + + + diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py new file mode 100644 index 0000000..721ee36 --- /dev/null +++ b/glymur/_uuid_io/__init__.py @@ -0,0 +1 @@ +from .Exif import _Exif diff --git a/glymur/jp2box.py b/glymur/jp2box.py index a270424..3ea911e 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -39,6 +39,8 @@ from .core import _COLOR_TYPE_MAP_DISPLAY from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .core import ANY_ICC_PROFILE, VENDOR_COLOR_METHOD +from ._uuid_io import _Exif + _METHOD_DISPLAY = { ENUMERATED_COLORSPACE: 'enumerated colorspace', RESTRICTED_ICC_PROFILE: 'restricted ICC profile', @@ -2084,7 +2086,7 @@ class UUIDBox(Jp2kBox): self.data = ET.ElementTree(elt) self._type = 'XMP' elif the_uuid.bytes == b'JpgTiffExif->JP2': - exif_obj = Exif(raw_data) + exif_obj = _Exif(raw_data) ifds = OrderedDict() ifds['Image'] = exif_obj.exif_image ifds['Photo'] = exif_obj.exif_photo @@ -2170,511 +2172,6 @@ class UUIDBox(Jp2kBox): return box -class Exif(object): - """ - Attributes - ---------- - read_buffer : bytes - Raw byte stream consisting of the UUID data. - endian : str - Either '<' for big-endian, or '>' for little-endian. - """ - - def __init__(self, read_buffer): - """Interpret raw buffer consisting of Exif IFD. - """ - self.exif_image = None - self.exif_photo = None - self.exif_gpsinfo = None - self.exif_iop = None - - self.read_buffer = read_buffer - - # Ignore the first six bytes. - # Next 8 should be (73, 73, 42, 8) - data = struct.unpack('' for little-endian. - num_tags : int - Number of tags in the IFD. - raw_ifd : dictionary - Maps tag number to "mildly-interpreted" tag value. - processed_ifd : dictionary - Maps tag name to "mildly-interpreted" tag value. - """ - datatype2fmt = {1: ('B', 1), - 2: ('B', 1), - 3: ('H', 2), - 4: ('I', 4), - 5: ('II', 8), - 7: ('B', 1), - 9: ('i', 4), - 10: ('ii', 8)} - - def __init__(self, endian, read_buffer, offset): - self.endian = endian - self.read_buffer = read_buffer - self.processed_ifd = OrderedDict() - - self.num_tags, = struct.unpack(endian + 'H', - read_buffer[offset:offset + 2]) - - fmt = self.endian + 'HHII' * self.num_tags - ifd_buffer = read_buffer[offset + 2:offset + 2 + self.num_tags * 12] - data = struct.unpack(fmt, ifd_buffer) - self.raw_ifd = OrderedDict() - for j, tag in enumerate(data[0::4]): - # The offset to the tag offset/payload is the offset to the IFD - # plus 2 bytes for the number of tags plus 12 bytes for each - # tag entry plus 8 bytes to the offset/payload itself. - toffp = read_buffer[offset + 10 + j * 12:offset + 10 + j * 12 + 4] - tag_data = self.parse_tag(data[j * 4 + 1], - data[j * 4 + 2], - toffp) - self.raw_ifd[tag] = tag_data - - def parse_tag(self, dtype, count, offset_buf): - """Interpret an Exif image tag data payload. - """ - fmt = self.datatype2fmt[dtype][0] * count - payload_size = self.datatype2fmt[dtype][1] * count - - if payload_size <= 4: - # Interpret the payload from the 4 bytes in the tag entry. - target_buffer = offset_buf[:payload_size] - else: - # Interpret the payload at the offset specified by the 4 bytes in - # the tag entry. - offset, = struct.unpack(self.endian + 'I', offset_buf) - target_buffer = self.read_buffer[offset:offset + payload_size] - - if dtype == 2: - # ASCII - if sys.hexversion < 0x03000000: - payload = target_buffer.rstrip('\x00') - else: - payload = target_buffer.decode('utf-8').rstrip('\x00') - - else: - payload = struct.unpack(self.endian + fmt, target_buffer) - if dtype == 5 or dtype == 10: - # Rational or Signed Rational. Construct the list of values. - rational_payload = [] - for j in range(count): - value = float(payload[j * 2]) / float(payload[j * 2 + 1]) - rational_payload.append(value) - payload = rational_payload - if count == 1: - # If just a single value, then return a scalar instead of a - # tuple. - payload = payload[0] - - return payload - - def post_process(self, tagnum2name): - """Map the tag name instead of tag number to the tag value. - """ - for tag, value in self.raw_ifd.items(): - try: - tag_name = tagnum2name[tag] - except KeyError: - # Ok, we don't recognize this tag. Just use the numeric id. - msg = 'Unrecognized Exif tag "{0}".'.format(tag) - warnings.warn(msg, UserWarning) - tag_name = tag - self.processed_ifd[tag_name] = value - - -class _ExifImageIfd(_Ifd): - """ - Attributes - ---------- - tagnum2name : dict - Maps Exif image tag numbers to the tag names. - ifd : dict - Maps tag names to tag values. - """ - tagnum2name = {11: 'ProcessingSoftware', - 254: 'NewSubfileType', - 255: 'SubfileType', - 256: 'ImageWidth', - 257: 'ImageLength', - 258: 'BitsPerSample', - 259: 'Compression', - 262: 'PhotometricInterpretation', - 263: 'Threshholding', - 264: 'CellWidth', - 265: 'CellLength', - 266: 'FillOrder', - 269: 'DocumentName', - 270: 'ImageDescription', - 271: 'Make', - 272: 'Model', - 273: 'StripOffsets', - 274: 'Orientation', - 277: 'SamplesPerPixel', - 278: 'RowsPerStrip', - 279: 'StripByteCounts', - 282: 'XResolution', - 283: 'YResolution', - 284: 'PlanarConfiguration', - 290: 'GrayResponseUnit', - 291: 'GrayResponseCurve', - 292: 'T4Options', - 293: 'T6Options', - 296: 'ResolutionUnit', - 301: 'TransferFunction', - 305: 'Software', - 306: 'DateTime', - 315: 'Artist', - 316: 'HostComputer', - 317: 'Predictor', - 318: 'WhitePoint', - 319: 'PrimaryChromaticities', - 320: 'ColorMap', - 321: 'HalftoneHints', - 322: 'TileWidth', - 323: 'TileLength', - 324: 'TileOffsets', - 325: 'TileByteCounts', - 330: 'SubIFDs', - 332: 'InkSet', - 333: 'InkNames', - 334: 'NumberOfInks', - 336: 'DotRange', - 337: 'TargetPrinter', - 338: 'ExtraSamples', - 339: 'SampleFormat', - 340: 'SMinSampleValue', - 341: 'SMaxSampleValue', - 342: 'TransferRange', - 343: 'ClipPath', - 344: 'XClipPathUnits', - 345: 'YClipPathUnits', - 346: 'Indexed', - 347: 'JPEGTables', - 351: 'OPIProxy', - 512: 'JPEGProc', - 513: 'JPEGInterchangeFormat', - 514: 'JPEGInterchangeFormatLength', - 515: 'JPEGRestartInterval', - 517: 'JPEGLosslessPredictors', - 518: 'JPEGPointTransforms', - 519: 'JPEGQTables', - 520: 'JPEGDCTables', - 521: 'JPEGACTables', - 529: 'YCbCrCoefficients', - 530: 'YCbCrSubSampling', - 531: 'YCbCrPositioning', - 532: 'ReferenceBlackWhite', - 700: 'XMLPacket', - 18246: 'Rating', - 18249: 'RatingPercent', - 32781: 'ImageID', - 33421: 'CFARepeatPatternDim', - 33422: 'CFAPattern', - 33423: 'BatteryLevel', - 33432: 'Copyright', - 33434: 'ExposureTime', - 33437: 'FNumber', - 33723: 'IPTCNAA', - 34377: 'ImageResources', - 34665: 'ExifTag', - 34675: 'InterColorProfile', - 34850: 'ExposureProgram', - 34852: 'SpectralSensitivity', - 34853: 'GPSTag', - 34855: 'ISOSpeedRatings', - 34856: 'OECF', - 34857: 'Interlace', - 34858: 'TimeZoneOffset', - 34859: 'SelfTimerMode', - 36867: 'DateTimeOriginal', - 37122: 'CompressedBitsPerPixel', - 37377: 'ShutterSpeedValue', - 37378: 'ApertureValue', - 37379: 'BrightnessValue', - 37380: 'ExposureBiasValue', - 37381: 'MaxApertureValue', - 37382: 'SubjectDistance', - 37383: 'MeteringMode', - 37384: 'LightSource', - 37385: 'Flash', - 37386: 'FocalLength', - 37387: 'FlashEnergy', - 37388: 'SpatialFrequencyResponse', - 37389: 'Noise', - 37390: 'FocalPlaneXResolution', - 37391: 'FocalPlaneYResolution', - 37392: 'FocalPlaneResolutionUnit', - 37393: 'ImageNumber', - 37394: 'SecurityClassification', - 37395: 'ImageHistory', - 37396: 'SubjectLocation', - 37397: 'ExposureIndex', - 37398: 'TIFFEPStandardID', - 37399: 'SensingMethod', - 40091: 'XPTitle', - 40092: 'XPComment', - 40093: 'XPAuthor', - 40094: 'XPKeywords', - 40095: 'XPSubject', - 50341: 'PrintImageMatching', - 50706: 'DNGVersion', - 50707: 'DNGBackwardVersion', - 50708: 'UniqueCameraModel', - 50709: 'LocalizedCameraModel', - 50710: 'CFAPlaneColor', - 50711: 'CFALayout', - 50712: 'LinearizationTable', - 50713: 'BlackLevelRepeatDim', - 50714: 'BlackLevel', - 50715: 'BlackLevelDeltaH', - 50716: 'BlackLevelDeltaV', - 50717: 'WhiteLevel', - 50718: 'DefaultScale', - 50719: 'DefaultCropOrigin', - 50720: 'DefaultCropSize', - 50721: 'ColorMatrix1', - 50722: 'ColorMatrix2', - 50723: 'CameraCalibration1', - 50724: 'CameraCalibration2', - 50725: 'ReductionMatrix1', - 50726: 'ReductionMatrix2', - 50727: 'AnalogBalance', - 50728: 'AsShotNeutral', - 50729: 'AsShotWhiteXY', - 50730: 'BaselineExposure', - 50731: 'BaselineNoise', - 50732: 'BaselineSharpness', - 50733: 'BayerGreenSplit', - 50734: 'LinearResponseLimit', - 50735: 'CameraSerialNumber', - 50736: 'LensInfo', - 50737: 'ChromaBlurRadius', - 50738: 'AntiAliasStrength', - 50739: 'ShadowScale', - 50740: 'DNGPrivateData', - 50741: 'MakerNoteSafety', - 50778: 'CalibrationIlluminant1', - 50779: 'CalibrationIlluminant2', - 50780: 'BestQualityScale', - 50781: 'RawDataUniqueID', - 50827: 'OriginalRawFileName', - 50828: 'OriginalRawFileData', - 50829: 'ActiveArea', - 50830: 'MaskedAreas', - 50831: 'AsShotICCProfile', - 50832: 'AsShotPreProfileMatrix', - 50833: 'CurrentICCProfile', - 50834: 'CurrentPreProfileMatrix', - 50879: 'ColorimetricReference', - 50931: 'CameraCalibrationSignature', - 50932: 'ProfileCalibrationSignature', - 50934: 'AsShotProfileName', - 50935: 'NoiseReductionApplied', - 50936: 'ProfileName', - 50937: 'ProfileHueSatMapDims', - 50938: 'ProfileHueSatMapData1', - 50939: 'ProfileHueSatMapData2', - 50940: 'ProfileToneCurve', - 50941: 'ProfileEmbedPolicy', - 50942: 'ProfileCopyright', - 50964: 'ForwardMatrix1', - 50965: 'ForwardMatrix2', - 50966: 'PreviewApplicationName', - 50967: 'PreviewApplicationVersion', - 50968: 'PreviewSettingsName', - 50969: 'PreviewSettingsDigest', - 50970: 'PreviewColorSpace', - 50971: 'PreviewDateTime', - 50972: 'RawImageDigest', - 50973: 'OriginalRawFileDigest', - 50974: 'SubTileBlockSize', - 50975: 'RowInterleaveFactor', - 50981: 'ProfileLookTableDims', - 50982: 'ProfileLookTableData', - 51008: 'OpcodeList1', - 51009: 'OpcodeList2', - 51022: 'OpcodeList3', - 51041: 'NoiseProfile'} - - def __init__(self, endian, read_buffer, offset): - _Ifd.__init__(self, endian, read_buffer, offset) - self.post_process(self.tagnum2name) - - -class _ExifPhotoIfd(_Ifd): - """Represents tags found in the Exif sub ifd. - """ - tagnum2name = {33434: 'ExposureTime', - 33437: 'FNumber', - 34850: 'ExposureProgram', - 34852: 'SpectralSensitivity', - 34855: 'ISOSpeedRatings', - 34856: 'OECF', - 34864: 'SensitivityType', - 34865: 'StandardOutputSensitivity', - 34866: 'RecommendedExposureIndex', - 34867: 'ISOSpeed', - 34868: 'ISOSpeedLatitudeyyy', - 34869: 'ISOSpeedLatitudezzz', - 36864: 'ExifVersion', - 36867: 'DateTimeOriginal', - 36868: 'DateTimeDigitized', - 37121: 'ComponentsConfiguration', - 37122: 'CompressedBitsPerPixel', - 37377: 'ShutterSpeedValue', - 37378: 'ApertureValue', - 37379: 'BrightnessValue', - 37380: 'ExposureBiasValue', - 37381: 'MaxApertureValue', - 37382: 'SubjectDistance', - 37383: 'MeteringMode', - 37384: 'LightSource', - 37385: 'Flash', - 37386: 'FocalLength', - 37396: 'SubjectArea', - 37500: 'MakerNote', - 37510: 'UserComment', - 37520: 'SubSecTime', - 37521: 'SubSecTimeOriginal', - 37522: 'SubSecTimeDigitized', - 40960: 'FlashpixVersion', - 40961: 'ColorSpace', - 40962: 'PixelXDimension', - 40963: 'PixelYDimension', - 40964: 'RelatedSoundFile', - 40965: 'InteroperabilityTag', - 41483: 'FlashEnergy', - 41484: 'SpatialFrequencyResponse', - 41486: 'FocalPlaneXResolution', - 41487: 'FocalPlaneYResolution', - 41488: 'FocalPlaneResolutionUnit', - 41492: 'SubjectLocation', - 41493: 'ExposureIndex', - 41495: 'SensingMethod', - 41728: 'FileSource', - 41729: 'SceneType', - 41730: 'CFAPattern', - 41985: 'CustomRendered', - 41986: 'ExposureMode', - 41987: 'WhiteBalance', - 41988: 'DigitalZoomRatio', - 41989: 'FocalLengthIn35mmFilm', - 41990: 'SceneCaptureType', - 41991: 'GainControl', - 41992: 'Contrast', - 41993: 'Saturation', - 41994: 'Sharpness', - 41995: 'DeviceSettingDescription', - 41996: 'SubjectDistanceRange', - 42016: 'ImageUniqueID', - 42032: 'CameraOwnerName', - 42033: 'BodySerialNumber', - 42034: 'LensSpecification', - 42035: 'LensMake', - 42036: 'LensModel', - 42037: 'LensSerialNumber'} - - def __init__(self, endian, read_buffer, offset): - _Ifd.__init__(self, endian, read_buffer, offset) - self.post_process(self.tagnum2name) - - -class _ExifGPSInfoIfd(_Ifd): - """Represents information found in the GPSInfo sub IFD. - """ - tagnum2name = {0: 'GPSVersionID', - 1: 'GPSLatitudeRef', - 2: 'GPSLatitude', - 3: 'GPSLongitudeRef', - 4: 'GPSLongitude', - 5: 'GPSAltitudeRef', - 6: 'GPSAltitude', - 7: 'GPSTimeStamp', - 8: 'GPSSatellites', - 9: 'GPSStatus', - 10: 'GPSMeasureMode', - 11: 'GPSDOP', - 12: 'GPSSpeedRef', - 13: 'GPSSpeed', - 14: 'GPSTrackRef', - 15: 'GPSTrack', - 16: 'GPSImgDirectionRef', - 17: 'GPSImgDirection', - 18: 'GPSMapDatum', - 19: 'GPSDestLatitudeRef', - 20: 'GPSDestLatitude', - 21: 'GPSDestLongitudeRef', - 22: 'GPSDestLongitude', - 23: 'GPSDestBearingRef', - 24: 'GPSDestBearing', - 25: 'GPSDestDistanceRef', - 26: 'GPSDestDistance', - 27: 'GPSProcessingMethod', - 28: 'GPSAreaInformation', - 29: 'GPSDateStamp', - 30: 'GPSDifferential'} - - def __init__(self, endian, read_buffer, offset): - _Ifd.__init__(self, endian, read_buffer, offset) - self.post_process(self.tagnum2name) - - -class _ExifInteroperabilityIfd(_Ifd): - """Represents tags found in the Interoperability sub IFD. - """ - tagnum2name = {1: 'InteroperabilityIndex', - 2: 'InteroperabilityVersion', - 4096: 'RelatedImageFileFormat', - 4097: 'RelatedImageWidth', - 4098: 'RelatedImageLength'} - - def __init__(self, endian, read_buffer, offset): - _Ifd.__init__(self, endian, read_buffer, offset) - self.post_process(self.tagnum2name) - - # Map each box ID to the corresponding class. _BOX_WITH_ID = { 'asoc': AssociationBox, diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py new file mode 100644 index 0000000..6278f57 --- /dev/null +++ b/glymur/test/test_jp2box_uuid.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +"""Test suite for printing. +""" +# C0302: don't care too much about having too many lines in a test module +# pylint: disable=C0302 + +# E061: unittest.mock introduced in 3.3 (python-2.7/pylint issue) +# pylint: disable=E0611,F0401 + +# R0904: Not too many methods in unittest. +# pylint: disable=R0904 + +import os +import re +import struct +import sys +import tempfile +import warnings +from xml.etree import cElementTree as ET + +if sys.hexversion < 0x02070000: + import unittest2 as unittest +else: + import unittest + +if sys.hexversion < 0x03000000: + from StringIO import StringIO +else: + from io import StringIO + +if sys.hexversion <= 0x03030000: + from mock import patch +else: + from unittest.mock import patch + +import glymur +from glymur import Jp2k +from .fixtures import OPJ_DATA_ROOT, opj_data_file, nemo_xmp_box + + +class TestUUIDExif(unittest.TestCase): + """Tests for UUIDs of Exif type.""" + + def setUp(self): + self.jp2file = glymur.data.nemo() + + def tearDown(self): + pass + + @unittest.skipIf(sys.hexversion < 0x03000000, "Requires assertWarns, 3.2+") + def test_unrecognized_exif_tag(self): + """Verify warning in case of unrecognized tag.""" + with tempfile.NamedTemporaryFile(suffix='.jp2', mode='wb') as tfile: + + with open(self.jp2file, 'rb') as ifptr: + tfile.write(ifptr.read()) + + # Write L, T, UUID identifier. + tfile.write(struct.pack('>I4s', 52, b'uuid')) + tfile.write(b'JpgTiffExif->JP2') + + tfile.write(b'Exif\x00\x00') + xbuffer = struct.pack(' Date: Sat, 26 Oct 2013 16:51:07 -0400 Subject: [PATCH 04/83] Refactored UUID handling. #104 New classes for each type in _uuid_io sub package. --- glymur/_uuid_io/Exif.py | 50 ++++++++++++------- glymur/_uuid_io/XMP.py | 46 +++++++++++++++++ glymur/_uuid_io/__init__.py | 5 +- glymur/_uuid_io/generic.py | 27 ++++++++++ glymur/core.py | 45 +++++++++++++++++ glymur/jp2box.py | 95 ++++++------------------------------ glymur/test/test_jp2k.py | 2 +- glymur/test/test_printing.py | 3 +- 8 files changed, 173 insertions(+), 100 deletions(-) create mode 100644 glymur/_uuid_io/XMP.py create mode 100644 glymur/_uuid_io/generic.py diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index f70a090..3fb24b1 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """ -Handlers for various UUID types. +Handlers for Exif UUIDs. Be nice if we would find a standard for this. """ +import pprint import struct import sys import warnings @@ -12,7 +13,7 @@ if sys.hexversion < 0x02070000: else: from collections import OrderedDict -class _Exif(object): +class UUIDExif(object): """ Attributes ---------- @@ -25,10 +26,10 @@ class _Exif(object): def __init__(self, read_buffer): """Interpret raw buffer consisting of Exif IFD. """ - self.exif_image = None - self.exif_photo = None - self.exif_gpsinfo = None - self.exif_iop = None + exif_image = None + exif_photo = None + exif_gpsinfo = None + exif_iop = None self.read_buffer = read_buffer @@ -45,24 +46,39 @@ class _Exif(object): # This is the 'Exif Image' portion. exif = _ExifImageIfd(self.endian, read_buffer[6:], offset) - self.exif_image = exif.processed_ifd + exif_image = exif.processed_ifd - if 'ExifTag' in self.exif_image.keys(): - offset = self.exif_image['ExifTag'] - photo = _ExifPhotoIfd(self.endian, read_buffer[6:], offset) - self.exif_photo = photo.processed_ifd + if 'ExifTag' in exif_image.keys(): + offset = exif_image['ExifTag'] + photo_ifd = _ExifPhotoIfd(self.endian, read_buffer[6:], offset) + exif_photo = photo_ifd.processed_ifd - if 'InteroperabilityTag' in self.exif_photo.keys(): - offset = self.exif_photo['InteroperabilityTag'] + if 'InteroperabilityTag' in exif_photo.keys(): + offset = exif_photo['InteroperabilityTag'] interop = _ExifInteroperabilityIfd(self.endian, read_buffer[6:], offset) - self.iop = interop.processed_ifd + iop = interop.processed_ifd - if 'GPSTag' in self.exif_image.keys(): - offset = self.exif_image['GPSTag'] + if 'GPSTag' in exif_image.keys(): + offset = exif_image['GPSTag'] gps = _ExifGPSInfoIfd(self.endian, read_buffer[6:], offset) - self.exif_gpsinfo = gps.processed_ifd + exif_gpsinfo = gps.processed_ifd + + self.ifds = OrderedDict() + self.ifds['Image'] = exif_image + self.ifds['Photo'] = exif_photo + self.ifds['GPSInfo'] = exif_gpsinfo + self.ifds['Iop'] = exif_iop + + def __str__(self): + # 2.7 has trouble pretty-printing ordered dicts, so print them + # as regular dicts. Not ideal, but at least it's good on 3.3+. + if sys.hexversion < 0x03000000: + data = dict(self.ifds) + else: + data = self.ifds + return '\n' + pprint.pformat(data) class _Ifd(object): diff --git a/glymur/_uuid_io/XMP.py b/glymur/_uuid_io/XMP.py new file mode 100644 index 0000000..0451a92 --- /dev/null +++ b/glymur/_uuid_io/XMP.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- + +""" +Handler for a UUID for XMP. +""" + +import sys +from xml.etree import cElementTree as ET + +from ..core import _pretty_print_xml + +class UUIDXMP(object): + """ + Handler for a UUID for XMP. + + Attributes + ---------- + packet : ElementTree + XML conforming to the XMP specifications. + + References + ---------- + .. [XMP] International Organization for Standardication. ISO/IEC + 16684-1:2012 - Graphic technology -- Extensible metadata platform (XMP) + specification -- Part 1: Data model, serialization and core properties + """ + def __init__(self, read_buffer): + """ + Parameters + ---------- + read_buffer : byte array + sequence of bytes that can be decoded into an XMP packet. + """ + + # XMP data. Parse as XML. + if sys.hexversion < 0x03000000: + # 2.x strings same as bytes + elt = ET.fromstring(read_buffer) + else: + # 3.x takes strings, not bytes. + text = read_buffer.decode('utf-8') + elt = ET.fromstring(text) + self.packet = ET.ElementTree(elt) + + def __str__(self): + return _pretty_print_xml(self.packet) diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py index 721ee36..5545351 100644 --- a/glymur/_uuid_io/__init__.py +++ b/glymur/_uuid_io/__init__.py @@ -1 +1,4 @@ -from .Exif import _Exif +from .Exif import UUIDExif +from .XMP import UUIDXMP +from .generic import UUIDGeneric + diff --git a/glymur/_uuid_io/generic.py b/glymur/_uuid_io/generic.py new file mode 100644 index 0000000..bad68a2 --- /dev/null +++ b/glymur/_uuid_io/generic.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +""" +Handler for a generic UUID. +""" + +class UUIDGeneric(object): + """ + Handler for a generic UUID that is not currently recognized. + + Attributes + ---------- + data : byte array + Sequence of uninterpreted bytes as read from the file. + """ + def __init__(self, read_buffer): + """ + Parameters + ---------- + read_buffer : byte array + sequence of bytes as read from the file. + """ + self.data = read_buffer + + def __str__(self): + return '{0} bytes'.format(len(self.data)) + diff --git a/glymur/core.py b/glymur/core.py index 22b5a19..620d0e3 100644 --- a/glymur/core.py +++ b/glymur/core.py @@ -1,5 +1,8 @@ """Core definitions to be shared amongst the modules. """ +import copy +import xml.etree.cElementTree as ET + # Progression order LRCP = 0 RLCP = 1 @@ -73,3 +76,45 @@ _CAPABILITIES_DISPLAY = { 1: '0', 2: '1', 3: '3'} + + +def _pretty_print_xml(xml, level=0): + """Pretty print XML data. + """ + xml = copy.deepcopy(xml) + _indent(xml.getroot(), level=level) + xmltext = ET.tostring(xml.getroot(), encoding='utf-8').decode('utf-8') + + # Indent it a bit. + lst = [(' ' + x) for x in xmltext.split('\n')] + try: + xml = '\n'.join(lst) + return '\n{0}'.format(xml) + except UnicodeEncodeError: + # This can happen on python 2.x if the character set contains certain + # non-ascii characters. Just print out the corresponding xml char + # entities instead. + xml = u'\n'.join(lst) + text = u'\n{0}'.format(xml) + text = text.encode('ascii', 'xmlcharrefreplace') + return text + + +def _indent(elem, level=0): + """Recipe for pretty printing XML. Please see + + http://effbot.org/zone/element-lib.htm#prettyprint + """ + i = "\n" + level * " " + if len(elem): + if not elem.text or not elem.text.strip(): + elem.text = i + " " + if not elem.tail or not elem.tail.strip(): + elem.tail = i + for elem in elem: + _indent(elem, level + 1) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + else: + if level and (not elem.tail or not elem.tail.strip()): + elem.tail = i diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 3ea911e..681cf51 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -13,7 +13,6 @@ References # pylint: disable=C0302,R0903,R0913 -import copy import datetime import math import os @@ -38,8 +37,9 @@ from .core import _COLORSPACE_MAP_DISPLAY from .core import _COLOR_TYPE_MAP_DISPLAY from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .core import ANY_ICC_PROFILE, VENDOR_COLOR_METHOD +from .core import _pretty_print_xml -from ._uuid_io import _Exif +from . import _uuid_io _METHOD_DISPLAY = { ENUMERATED_COLORSPACE: 'enumerated colorspace', @@ -2065,8 +2065,11 @@ class UUIDBox(Jp2kBox): ---------- the_uuid : uuid.UUID Identifies the type of UUID box. + data : object + Specific to each type of UUID. There are handlers for XMP, Exif, + and unknown UUIDs. raw_data : byte array - This is the "payload" of data for the specified UUID. + Sequence of uninterpreted bytes as read from the file. length : int length of the box in bytes. offset : int @@ -2076,59 +2079,33 @@ class UUIDBox(Jp2kBox): self.uuid = the_uuid if the_uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - # XMP data. Parse as XML. Seems to be a difference between - # ElementTree in version 2.7 and 3.3. - if sys.hexversion < 0x03000000: - elt = ET.fromstring(raw_data) - else: - text = raw_data.decode('utf-8') - elt = ET.fromstring(text) - self.data = ET.ElementTree(elt) + self.data = _uuid_io.UUIDXMP(raw_data) self._type = 'XMP' elif the_uuid.bytes == b'JpgTiffExif->JP2': - exif_obj = _Exif(raw_data) - ifds = OrderedDict() - ifds['Image'] = exif_obj.exif_image - ifds['Photo'] = exif_obj.exif_photo - ifds['GPSInfo'] = exif_obj.exif_gpsinfo - ifds['Iop'] = exif_obj.exif_iop - self.data = ifds + self.data = _uuid_io.UUIDExif(raw_data) self._type = 'Exif' else: - self.data = raw_data + self.data = _uuid_io.UUIDGeneric(raw_data) self._type = 'unknown' + + self.raw_data = raw_data self.length = length self.offset = offset def __str__(self): msg = '{0}\n' - msg += ' UUID: {1}{2}\n' + msg += ' UUID: {1} ({2})\n' msg += ' UUID Data: {3}' - if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - uuid_type = ' (XMP)' - uuid_data = _pretty_print_xml(self.data) - elif self.uuid.bytes == b'JpgTiffExif->JP2': - uuid_type = ' (Exif)' - # 2.7 has trouble pretty-printing ordered dicts, so print them - # as regular dicts. Not ideal, but at least it's good on 3.3+. - if sys.hexversion < 0x03000000: - data = dict(self.data) - else: - data = self.data - uuid_data = '\n' + pprint.pformat(data) - else: - uuid_type = '' - uuid_data = '{0} bytes'.format(len(self.data)) - msg = msg.format(Jp2kBox.__str__(self), self.uuid, - uuid_type, - uuid_data) + self._type, + str(self.data)) return msg + def write(self, fptr): """Write a UUID box box to file. """ @@ -2196,45 +2173,3 @@ _BOX_WITH_ID = { 'url ': DataEntryURLBox, 'uuid': UUIDBox, 'xml ': XMLBox} - - -def _indent(elem, level=0): - """Recipe for pretty printing XML. Please see - - http://effbot.org/zone/element-lib.htm#prettyprint - """ - i = "\n" + level * " " - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - _indent(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i - - -def _pretty_print_xml(xml, level=0): - """Pretty print XML data. - """ - xml = copy.deepcopy(xml) - _indent(xml.getroot(), level=level) - xmltext = ET.tostring(xml.getroot(), encoding='utf-8').decode('utf-8') - - # Indent it a bit. - lst = [(' ' + x) for x in xmltext.split('\n')] - try: - xml = '\n'.join(lst) - return '\n{0}'.format(xml) - except UnicodeEncodeError: - # This can happen on python 2.x if the character set contains certain - # non-ascii characters. Just print out the corresponding xml char - # entities instead. - xml = u'\n'.join(lst) - text = u'\n{0}'.format(xml) - text = text.encode('ascii', 'xmlcharrefreplace') - return text diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index c9e750a..026b129 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -353,7 +353,7 @@ class TestJp2k(unittest.TestCase): def test_xmp_attribute(self): """Verify the XMP packet in the shipping example file can be read.""" j = Jp2k(self.jp2file) - xmp = j.box[3].data + xmp = j.box[3].data.packet ns0 = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' ns2 = '{http://ns.adobe.com/xap/1.0/}' name = '{0}RDF/{0}Description/{1}CreatorTool'.format(ns0, ns2) diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 5824e0e..a1f0c55 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -636,6 +636,7 @@ class TestPrinting(unittest.TestCase): actual = fake_out.getvalue().strip() expected = nemo_xmp_box + self.maxDiff = None self.assertEqual(actual, expected) def test_codestream(self): @@ -1024,7 +1025,7 @@ class TestPrinting(unittest.TestCase): print(jp2.box[4]) actual = fake_out.getvalue().strip() lines = ['UUID Box (uuid) @ (1544, 25)', - ' UUID: 3a0d0218-0ae9-4115-b376-4bca41ce0e71', + ' UUID: 3a0d0218-0ae9-4115-b376-4bca41ce0e71 (unknown)', ' UUID Data: 1 bytes'] expected = '\n'.join(lines) From 7e717b50370f56dd0895922196098ef6c72c347c Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 26 Oct 2013 18:05:44 -0400 Subject: [PATCH 05/83] Made Exif handling more resilient. #104 --- glymur/_uuid_io/Exif.py | 6 +++- glymur/jp2box.py | 23 +++++++++---- glymur/test/test_jp2box_uuid.py | 57 ++++++++++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index 3fb24b1..0a86c18 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -39,9 +39,13 @@ class UUIDExif(object): if data[0] == 73 and data[1] == 73: # little endian self.endian = '<' - else: + elif data[0] == 77 and data[1] == 77: # big endian self.endian = '>' + else: + msg = "Bad byte order indication: {0}".format(read_buffer[6:8]) + raise RuntimeError(msg) + offset = data[3] # This is the 'Exif Image' portion. diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 681cf51..074d18e 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -19,6 +19,7 @@ import os import pprint import struct import sys +import traceback import uuid import warnings import xml.etree.cElementTree as ET @@ -2078,15 +2079,23 @@ class UUIDBox(Jp2kBox): Jp2kBox.__init__(self, box_id='uuid', longname='UUID') self.uuid = the_uuid - if the_uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - self.data = _uuid_io.UUIDXMP(raw_data) - self._type = 'XMP' - elif the_uuid.bytes == b'JpgTiffExif->JP2': - self.data = _uuid_io.UUIDExif(raw_data) - self._type = 'Exif' - else: + try: + if the_uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): + self.data = _uuid_io.UUIDXMP(raw_data) + self._type = 'XMP' + elif the_uuid.bytes == b'JpgTiffExif->JP2': + self.data = _uuid_io.UUIDExif(raw_data) + self._type = 'Exif' + else: + self.data = _uuid_io.UUIDGeneric(raw_data) + self._type = 'unknown' + except Exception as err: + # In case of any exception, create the generic UUID. self.data = _uuid_io.UUIDGeneric(raw_data) self._type = 'unknown' + msg = "Error encountered during UUID processing, " + msg += "the UUID will be treated as generic.\n\n{0}" + warnings.warn(msg.format(traceback.format_exc())) self.raw_data = raw_data diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index 6278f57..a1d93a4 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -63,7 +63,7 @@ class TestUUIDExif(unittest.TestCase): xbuffer = struct.pack('I4s', 52, b'uuid')) + tfile.write(b'JpgTiffExif->JP2') + + tfile.write(b'Exif\x00\x00') + xbuffer = struct.pack('I4s', 52, b'uuid')) + tfile.write(b'JpgTiffExif->JP2') + + tfile.write(b'Exif\x00\x00') + xbuffer = struct.pack(' Date: Sat, 26 Oct 2013 18:21:24 -0400 Subject: [PATCH 06/83] Pylint issues, #104 --- glymur/_uuid_io/Exif.py | 2 +- glymur/_uuid_io/__init__.py | 4 +++- glymur/jp2box.py | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index 0a86c18..ccc03dd 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -62,7 +62,7 @@ class UUIDExif(object): interop = _ExifInteroperabilityIfd(self.endian, read_buffer[6:], offset) - iop = interop.processed_ifd + exif_iop = interop.processed_ifd if 'GPSTag' in exif_image.keys(): offset = exif_image['GPSTag'] diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py index 5545351..a23c2ce 100644 --- a/glymur/_uuid_io/__init__.py +++ b/glymur/_uuid_io/__init__.py @@ -1,4 +1,6 @@ +""" +Sub package for handling various UUIDs. +""" from .Exif import UUIDExif from .XMP import UUIDXMP from .generic import UUIDGeneric - diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 074d18e..34eeeb8 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2089,7 +2089,7 @@ class UUIDBox(Jp2kBox): else: self.data = _uuid_io.UUIDGeneric(raw_data) self._type = 'unknown' - except Exception as err: + except Exception: # In case of any exception, create the generic UUID. self.data = _uuid_io.UUIDGeneric(raw_data) self._type = 'unknown' @@ -2121,15 +2121,15 @@ class UUIDBox(Jp2kBox): if self._type != 'XMP': msg = "Only XMP UUID boxes can currently be written." raise NotImplementedError(msg) - serialized_buffer = b'' - serialized_buffer += ET.tostring(self.data.getroot(), encoding='utf-8') - serialized_buffer += b'' + serialized = b'' + serialized += ET.tostring(self.data.packet.getroot(), encoding='utf-8') + serialized += b'' if self.length == 0: - self.length = 24 + len(serialized_buffer) + self.length = 24 + len(serialized) read_buffer = struct.pack('>I4s', self.length, b'uuid') fptr.write(read_buffer) fptr.write(self.uuid.bytes) - fptr.write(serialized_buffer) + fptr.write(serialized) @staticmethod def parse(fptr, offset, length): From c4060f23c88ece87a5e97daccf1c7c37cbe80b88 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 26 Oct 2013 19:23:33 -0400 Subject: [PATCH 07/83] Added big endian Exif support. #104 --- glymur/_uuid_io/Exif.py | 6 +++--- glymur/test/test_jp2box_uuid.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index ccc03dd..fb5e2a1 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -34,8 +34,8 @@ class UUIDExif(object): self.read_buffer = read_buffer # Ignore the first six bytes. - # Next 8 should be (73, 73, 42, 8) - data = struct.unpack('I4s', 52, b'uuid')) + tfile.write(b'JpgTiffExif->JP2') + + tfile.write(b'Exif\x00\x00') + xbuffer = struct.pack('>BBHI', 77, 77, 42, 8) + tfile.write(xbuffer) + + # We will write just a single tag. + tfile.write(struct.pack('>H', 1)) + + # The "Make" tag is tag no. 271. + tfile.write(struct.pack('>HHI4s', 271, 2, 3, b'HTC\x00')) + tfile.flush() + + jp2 = glymur.Jp2k(tfile.name) + self.assertEqual(jp2.box[-1].data.ifds['Image']['Make'], "HTC") + if __name__ == "__main__": unittest.main() From 119dc0dfcd124332213270bea3dfcef19d5d9aa6 Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 27 Oct 2013 14:28:22 -0400 Subject: [PATCH 08/83] minor documentation tweaks. #104 --- glymur/jp2box.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index c02f497..ec1b349 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1209,7 +1209,7 @@ class JPEG2000SignatureBox(Jp2kBox): offset of the box from the start of the file. longname : str more verbose description of the box. - signature : byte + signature : tuple Four-byte tuple identifying the file as JPEG 2000. """ def __init__(self, signature=(13, 10, 135, 10), length=0, offset=-1): @@ -1807,7 +1807,7 @@ class LabelBox(Jp2kBox): longname : str more verbose description of the box. label : str - Label + Textual label. """ def __init__(self, label, length=0, offset=-1): Jp2kBox.__init__(self, box_id='lbl ', longname='Label') @@ -2179,9 +2179,12 @@ class UUIDBox(Jp2kBox): more verbose description of the box. uuid : uuid.UUID 16-byte UUID - data : bytes or dict or ElementTree.Element - Vendor-specific data. Exif UUIDs are interpreted as dictionaries. - XMP UUIDs are interpreted as standard XML. + raw_data : byte array + Sequence of uninterpreted bytes as read from the file. + data : object + Specific to each type of UUID. There are handlers for XMP, Exif, and + generic (unknown) UUIDs. In the case of XMP and Exif UUIDs, this is + the interpreted version of raw_data. References ---------- @@ -2195,9 +2198,6 @@ class UUIDBox(Jp2kBox): ---------- the_uuid : uuid.UUID Identifies the type of UUID box. - data : object - Specific to each type of UUID. There are handlers for XMP, Exif, - and unknown UUIDs. raw_data : byte array Sequence of uninterpreted bytes as read from the file. length : int From 727c4dd2ea357a5dc57d551a318cf08f0dc60ee7 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 27 Oct 2013 18:13:42 -0400 Subject: [PATCH 09/83] Verifying that we can write XMP UUIDs. --- glymur/test/fixtures.py | 16 ++++++++++++++++ glymur/test/test_jp2box_uuid.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 68c0706..9c543bf 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -254,3 +254,19 @@ nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) """ + +SimpleRDF = """ + + + Simple value + + + + Suse + Fedora + + + + +""" diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index e8cb7be..c17a6ab 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -12,9 +12,11 @@ import os import re +import shutil import struct import sys import tempfile +import uuid import warnings from xml.etree import cElementTree as ET @@ -35,9 +37,38 @@ else: import glymur from glymur import Jp2k -from .fixtures import OPJ_DATA_ROOT, opj_data_file, nemo_xmp_box +from .fixtures import OPJ_DATA_ROOT, opj_data_file, SimpleRDF +class TestUUIDXMP(unittest.TestCase): + """Tests for UUIDs of XMP type.""" + + def setUp(self): + self.jp2file = glymur.data.nemo() + + def tearDown(self): + pass + + def test_append(self): + """Should be able to append an XMP UUID box.""" + the_uuid = uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac') + raw_data = SimpleRDF.encode('utf-8') + with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile: + shutil.copyfile(self.jp2file, tfile.name) + jp2 = Jp2k(tfile.name) + ubox = glymur.jp2box.UUIDBox(the_uuid=the_uuid, raw_data=raw_data) + jp2.append(ubox) + + # Should be two UUID boxes now. + expected_ids = ['jP ', 'ftyp', 'jp2h', 'uuid', 'jp2c', 'uuid'] + actual_ids = [b.box_id for b in jp2.box] + self.assertEqual(actual_ids, expected_ids) + + # The data should be an XMP packet, which gets interpreted as + # an ElementTree. + self.assertTrue(isinstance(jp2.box[-1].data.packet, + ET.ElementTree)) + class TestUUIDExif(unittest.TestCase): """Tests for UUIDs of Exif type.""" From 32760a6ecc6322ecd746a15b5adbbad73f8608aa Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 27 Oct 2013 22:09:15 -0400 Subject: [PATCH 10/83] Doc updates for writing XMP UUIDs. #104 --- docs/source/how_do_i.rst | 84 ++++++++++++++++++++++-------------- docs/source/introduction.rst | 12 +++--- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index 212862d..443e8db 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -4,7 +4,7 @@ How do I...? ... read the lowest resolution thumbnail? -===================================== +========================================= Printing the Jp2k object should reveal the number of resolutions (look in the COD segment section), but you can take a shortcut by supplying -1 as the resolution level. :: @@ -15,7 +15,7 @@ resolution level. :: >>> thumbnail = j.read(rlevel=-1) ... display metadata? -================= +===================== There are two ways. From the unix command line, the script *jp2dump* is available. :: @@ -35,7 +35,7 @@ codestream box, only the main header is printed. It is possible to print >>> print(j.get_codestream()) ... add XML metadata? -================= +===================== You can append any number of XML boxes to a JP2 file (not to a raw codestream). Consider the following XML file `data.xml` : :: @@ -66,12 +66,12 @@ The **append** method can add an XML box as shown below:: >>> jp2.append(xmlbox) >>> print(jp2) -... add metadata in a more general fashion? -======================================= +... add even more metadata? +=========================== An existing raw codestream (or JP2 file) can be wrapped (re-wrapped) in a user-defined set of JP2 boxes. To get just a minimal JP2 jacket on the codestream provided by `goodstuff.j2k` (a file consisting of a raw codestream), -you can use the **wrap** method with no box argument: :: +you can use the :py:meth:`wrap` method with no box argument: :: >>> import glymur >>> jfile = glymur.data.goodstuff() @@ -108,8 +108,9 @@ JP2 header superbox. XML boxes are not in the minimal set of box requirements for the JP2 format, so in order to add an XML box into the mix before the codestream box, we'll need to -re-specify all of the boxes. If you already have a JP2 jacket in place, you can just reuse that, -though. Take the following example content in an XML file `favorites.xml` : :: +re-specify all of the boxes. If you already have a JP2 jacket in place, +you can just reuse that, though. Take the following example content in +an XML file `favorites.xml` : :: @@ -152,13 +153,13 @@ the following will work. :: . (truncated) . -As to the question of which method you should use, **append** or **wrap**, -to add metadata, you should keep in mind that **wrap** produces a new JP2 file, -while **append** modifies an existing file and is currently limited to XML -boxes. +As to the question of which method you should use, :py:meth:`append` or +:py:meth:`wrap`, to add metadata, you should keep in mind that :py:meth:`wrap` +produces a new JP2 file, while :py:meth:`append` modifies an existing file and +is currently limited to XML and UUID boxes. ... create an image with an alpha layer? -==================================== +======================================== OpenJPEG can create JP2 files with more than 3 components (requires the development version of OpenJPEG), but by default, any extra components are @@ -219,32 +220,49 @@ Here's how the Preview application on the mac shows the RGBA image. .. image:: goodstuff_alpha.png -work with XMP UUIDs? -==================== +... work with XMP UUIDs? +======================== The example JP2 file shipped with glymur has an XMP UUID. :: >>> import glymur >>> j = glymur.Jp2k(glymur.data.nemo()) - >>> print(j.box[4]) - UUID Box (uuid) @ (715, 2412) - UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - + >>> print(j.box[3]) # formatting added to the XML below + - - - + . + . + . + -Since the UUID data in this case is returned as an ElementTree instance, one can -use ElementTree to access the data. For example, to extract the -**CreatorTool** attribute value, the following would work:: +Since the UUID data in this case is returned as an ElementTree instance, +one can use ElementTree from the standard library to access the data. +For example, to extract the **CreatorTool** attribute value, the following +would work:: - >>> xmp = j.box[4].data - >>> ns0 = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' - >>> ns1 = '{http://ns.adobe.com/xap/1.0/}' - >>> name = '{0}RDF/{0}Description'.format(ns0) + >>> xmp = j.box[3].data.packet + >>> rdf = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' + >>> ns2 = '{http://ns.adobe.com/xap/1.0/}' + >>> name = '{0}RDF/{0}Description/{1}CreatorTool'.format(rdf, ns2) >>> elt = xmp.find(name) >>> elt - - >>> elt.attrib['{0}CreatorTool'.format(ns1)] - 'glymur' + + >>> elt.text + 'Google' + +Yes, that's painful. A better solution is to install the Python XMP Toolkit +(developer branch):: + + >>> from libxmp import XMPMeta + >>> from libxmp.consts import XMP_NS_XMP as NS_XAP + >>> meta = XMPMeta() + >>> meta.parse_from_str(j.box[3].raw_data.decode('utf-8')) + >>> meta.get_property(NS_XAP, 'CreatorTool') + 'Google' + diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index e1a3b3b..0a88902 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -7,11 +7,9 @@ which allows one to read and write JPEG 2000 files from within Python. Glymur supports both reading and writing of JPEG 2000 images, but writing JPEG 2000 images is currently limited to images that can fit in memory -Of particular focus is retrieval of metadata. Reading Exif UUIDs is supported, -as is reading XMP UUIDs as the XMP data packet is just XML. There is -some very limited support for reading JPX metadata. For instance, -**asoc** and **labl** boxes are recognized, so GMLJP2 metadata can -be retrieved from such JPX files. +In regards to metadata, most JP2 boxes are properly interpreted. +Certain optional JP2 boxes can also be written, including XML boxes and +XMP UUIDs. There is some very limited support for reading JPX metadata. Glymur works on Python 2.6, 2.7, and 3.3. @@ -20,8 +18,8 @@ OpenJPEG Installation Glymur will read JPEG 2000 images with versions 1.3, 1.4, 1.5, 2.0, and the trunk/development version of OpenJPEG. Writing images is only supported with the 1.5 or better, however, and the trunk/development -version is strongly recommended. For more information about OpenJPEG, -please consult http://www.openjpeg.org. +version of OpenJPEG is strongly recommended. For more information about +OpenJPEG, please consult http://www.openjpeg.org. If you use MacPorts or if you have a sufficiently recent version of Linux, your package manager should already provide you with a version of From ffe17f12cbb9545e0d5abcc4f57ffe48dd6ebdba Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 23 Jan 2014 21:50:48 -0500 Subject: [PATCH 11/83] Introducing python-xmp-toolkit requirement. #104 Down to 3 failures and 1 error. --- glymur/_uuid_io/Exif.py | 83 +++++++-------------------------- glymur/_uuid_io/XMP.py | 46 ------------------ glymur/_uuid_io/__init__.py | 6 +-- glymur/_uuid_io/generic.py | 27 ----------- glymur/jp2box.py | 77 ++++++++++++++---------------- glymur/jp2k.py | 4 +- glymur/test/test_jp2box_uuid.py | 5 +- glymur/test/test_jp2k.py | 12 ++--- setup.py | 2 +- 9 files changed, 67 insertions(+), 195 deletions(-) delete mode 100644 glymur/_uuid_io/XMP.py delete mode 100644 glymur/_uuid_io/generic.py diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index fb5e2a1..c1b30f0 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -13,76 +13,27 @@ if sys.hexversion < 0x02070000: else: from collections import OrderedDict -class UUIDExif(object): +def tiff_header(read_buffer): """ - Attributes - ---------- - read_buffer : bytes - Raw byte stream consisting of the UUID data. - endian : str - Either '<' for big-endian, or '>' for little-endian. """ + # Ignore the first six bytes. + # Next 8 should be (73, 73, 42, 8) or (77, 77, 42, 8) + data = struct.unpack('JP2': - self.data = _uuid_io.UUIDExif(raw_data) - self._type = 'Exif' - else: - self.data = _uuid_io.UUIDGeneric(raw_data) - self._type = 'unknown' - except Exception: - # In case of any exception, create the generic UUID. - self.data = _uuid_io.UUIDGeneric(raw_data) - self._type = 'unknown' - msg = "Error encountered during UUID processing, " - msg += "the UUID will be treated as generic.\n\n{0}" - warnings.warn(msg.format(traceback.format_exc())) - - self.raw_data = raw_data - self.length = length self.offset = offset + self.data = None + + try: + self._parse_raw_data() + except Exception as e: + warnings.warn(str(e)) + + def _parse_raw_data(self): + """ + Private function for parsing UUID payloads if possible. + """ + if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): + xmp = XMPMeta() + xmp.parse_from_str(self.raw_data.decode('utf-8'), + xmpmeta_wrap=False) + self.data = xmp + elif self.uuid.bytes == b'JpgTiffExif->JP2': + self.data = _uuid_io.tiff_header(self.raw_data) + else: + self.data = self.raw_data def __repr__(self): msg = "glymur.jp2box.UUIDBox(the_uuid={0}, " msg += "raw_data=)" - return msg.format(repr(self.uuid), len(self.raw_data)) - + return msg.format(repr(self.uuid), len(self.data)) def __str__(self): msg = '{0}\n' - msg += ' UUID: {1} ({2})\n' - msg += ' UUID Data: {3}' + msg += ' UUID: {1}\n' + msg += ' UUID Data: {2}' - msg = msg.format(Jp2kBox.__str__(self), - self.uuid, - self._type, - str(self.data)) + msg = msg.format(Jp2kBox.__str__(self), self.uuid, str(self.data)) return msg - def write(self, fptr): - """Write a UUID box box to file. + """Write a UUID box to file. """ - if self._type != 'XMP': + if self.uuid != uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): msg = "Only XMP UUID boxes can currently be written." raise NotImplementedError(msg) - serialized = b'' - serialized += ET.tostring(self.data.packet.getroot(), encoding='utf-8') - serialized += b'' - if self.length == 0: - self.length = 24 + len(serialized) - read_buffer = struct.pack('>I4s', self.length, b'uuid') - fptr.write(read_buffer) + write_buffer = struct.pack('>I4s', self.length, b'uuid') + fptr.write(write_buffer) fptr.write(self.uuid.bytes) - fptr.write(serialized) + fptr.write(self.raw_data) @staticmethod def parse(fptr, offset, length): diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 4d7071e..a337ae7 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -20,6 +20,7 @@ import ctypes import math import os import struct +from uuid import UUID import warnings import numpy as np @@ -526,7 +527,8 @@ class Jp2k(Jp2kBox): raise IOError(msg) if not ((box.box_id == 'xml ') or - (box.box_id == 'uuid' and box._type == 'XMP')): + (box.box_id == 'uuid' and + box.uuid == UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'))): msg = "Only XML boxes and XMP UUID boxes can currently be appended." raise IOError(msg) diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index c17a6ab..8f24f74 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -35,6 +35,8 @@ if sys.hexversion <= 0x03030000: else: from unittest.mock import patch +from libxmp import XMPMeta + import glymur from glymur import Jp2k from .fixtures import OPJ_DATA_ROOT, opj_data_file, SimpleRDF @@ -66,8 +68,7 @@ class TestUUIDXMP(unittest.TestCase): # The data should be an XMP packet, which gets interpreted as # an ElementTree. - self.assertTrue(isinstance(jp2.box[-1].data.packet, - ET.ElementTree)) + self.assertTrue(isinstance(jp2.box[-1].data, XMPMeta)) class TestUUIDExif(unittest.TestCase): """Tests for UUIDs of Exif type.""" diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index c38cacd..c055aba 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -30,6 +30,8 @@ import warnings import numpy as np import pkg_resources +import libxmp + import glymur from glymur import Jp2k @@ -362,13 +364,9 @@ class TestJp2k(unittest.TestCase): def test_xmp_attribute(self): """Verify the XMP packet in the shipping example file can be read.""" j = Jp2k(self.jp2file) - xmp = j.box[3].data.packet - ns0 = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' - ns2 = '{http://ns.adobe.com/xap/1.0/}' - name = '{0}RDF/{0}Description/{1}CreatorTool'.format(ns0, ns2) - elt = xmp.find(name) - self.assertEqual(elt.text, 'Google') - + xmp = j.box[3].data + creator_tool = xmp.get_property(libxmp.consts.XMP_NS_XMP, 'CreatorTool') + self.assertEqual(creator_tool, 'Google') @unittest.skipIf(re.match(r"""1\.[01234]""", glymur.version.openjpeg_version), "Requires at least version 1.5") diff --git a/setup.py b/setup.py index b780e4d..b4fdc2f 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ kwargs = {'name': 'Glymur', 'license': 'MIT', 'test_suite': 'glymur.test'} -instllrqrs = ['numpy>=1.4.1'] +instllrqrs = ['numpy>=1.4.1', 'python-xmp-toolkit>=2.0.0'] if sys.hexversion < 0x03030000: instllrqrs.append('contextlib2>=0.4') instllrqrs.append('mock>=1.0.1') From 553b36d40ee1bf3a043db39bdc4f68fa902c7565 Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 24 Jan 2014 11:08:23 -0500 Subject: [PATCH 12/83] Passing on Anaconda platform with python-xmp-toolkit 2.0. #104 --- glymur/_uuid_io/Exif.py | 32 ++++++ glymur/_uuid_io/__init__.py | 2 +- glymur/jp2box.py | 29 +++--- glymur/test/fixtures.py | 174 +++++++++++++++++--------------- glymur/test/test_jp2box_uuid.py | 2 +- 5 files changed, 138 insertions(+), 101 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index c1b30f0..a7c04aa 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -3,6 +3,7 @@ Handlers for Exif UUIDs. Be nice if we would find a standard for this. """ import pprint +import re import struct import sys import warnings @@ -13,8 +14,39 @@ if sys.hexversion < 0x02070000: else: from collections import OrderedDict +# The Python XMP Toolkit may be used for XMP UUIDs, but only if available and +# if the version is at least 2.0.0. +try: + import libxmp + if hasattr(libxmp, 'version') and re.match('[2-9].\d*.\d*', libxmp.version.VERSION): + from libxmp import XMPMeta + _HAS_PYTHON_XMP_TOOLKIT = True + else: + _HAS_PYTHON_XMP_TOOLKIT = False +except ImportError: + _HAS_PYTHON_XMP_TOOLKIT = False + +def xmp(read_buffer): + """ + If libxmp 2.0+ is installed, use it to describe the XMP data. + """ + if not _HAS_PYTHON_XMP_TOOLKIT: + # If the python xmp toolkit is not available or is not advanced enough, + # then issue a warning and just make available the raw data. + msg = "An XMP UUID was detected, but the Python XMP Toolkit package " + msg += "is either not available or is too old (must be at least 2.0). " + msg += "The UUID data field will consist only of the raw uninterpreted " + msg += "bytes." + warnings.warn(msg, UserWarning) + return read_buffer + + xmp = XMPMeta() + xmp.parse_from_str(read_buffer.decode('utf-8'), xmpmeta_wrap=False) + return xmp + def tiff_header(read_buffer): """ + Interpret the UUID data as a TIFF header. """ # Ignore the first six bytes. # Next 8 should be (73, 73, 42, 8) or (77, 77, 42, 8) diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py index d12d20c..89bbf21 100644 --- a/glymur/_uuid_io/__init__.py +++ b/glymur/_uuid_io/__init__.py @@ -1,4 +1,4 @@ """ Sub package for handling various types of UUIDs. """ -from .Exif import tiff_header +from .Exif import tiff_header, xmp diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 300ac12..8b0f6f3 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -33,12 +33,6 @@ else: import numpy as np -try: - from libxmp import XMPMeta - _HAS_PYTHON_XMP_TOOLKIT = True -except ImportError: - _HAS_PYTHON_XMP_TOOLKIT = False - from .codestream import Codestream from .core import _COLORSPACE_MAP_DISPLAY from .core import _COLOR_TYPE_MAP_DISPLAY @@ -2228,10 +2222,7 @@ class UUIDBox(Jp2kBox): Private function for parsing UUID payloads if possible. """ if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - xmp = XMPMeta() - xmp.parse_from_str(self.raw_data.decode('utf-8'), - xmpmeta_wrap=False) - self.data = xmp + self.data = _uuid_io.xmp(self.raw_data) elif self.uuid.bytes == b'JpgTiffExif->JP2': self.data = _uuid_io.tiff_header(self.raw_data) else: @@ -2243,11 +2234,19 @@ class UUIDBox(Jp2kBox): return msg.format(repr(self.uuid), len(self.data)) def __str__(self): - msg = '{0}\n' - msg += ' UUID: {1}\n' - msg += ' UUID Data: {2}' - - msg = msg.format(Jp2kBox.__str__(self), self.uuid, str(self.data)) + if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): + utype = "XMP" + elif self.uuid.bytes == b'JpgTiffExif->JP2': + utype = "EXIF" + else: + utype = "unknown" + msg = '{0}\n UUID: {1} ({2})\n'.format(Jp2kBox.__str__(self), + self.uuid, + utype) + if utype == 'unknown': + msg += ' UUID Data: {0} bytes'.format(len(self.raw_data)) + else: + msg += ' UUID Data: {0}'.format(str(self.data)) return msg diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 9c543bf..a33d1d6 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -170,90 +170,96 @@ def read_pgx_header(pgx_file): nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - """ + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + +""" SimpleRDF = """ Date: Fri, 24 Jan 2014 15:35:23 -0500 Subject: [PATCH 13/83] Tests passing on bare 2.6.6 python. #104 --- glymur/test/fixtures.py | 12 ++++++++++++ glymur/test/test_jp2box_uuid.py | 14 ++++++++++---- glymur/test/test_jp2k.py | 9 ++++++--- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index a33d1d6..164f93d 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -11,6 +11,18 @@ import numpy as np import glymur +# The Python XMP Toolkit may be used for XMP UUIDs, but only if available and +# if the version is at least 2.0.0. +try: + import libxmp + if hasattr(libxmp, 'version') and re.match('[2-9].\d*.\d*', libxmp.version.VERSION): + from libxmp import XMPMeta + HAS_PYTHON_XMP_TOOLKIT = True + else: + HAS_PYTHON_XMP_TOOLKIT = False +except ImportError: + HAS_PYTHON_XMP_TOOLKIT = False + # Need to know of the libopenjp2 version is the official 2.0.0 release and NOT # the 2.0+ development version. OPENJP2_IS_V2_OFFICIAL = False diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index 3e7f196..1415777 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -35,7 +35,9 @@ if sys.hexversion <= 0x03030000: else: from unittest.mock import patch -from libxmp import XMPMeta +from .fixtures import HAS_PYTHON_XMP_TOOLKIT, OPJ_DATA_ROOT +if HAS_PYTHON_XMP_TOOLKIT: + from libxmp import XMPMeta import glymur from glymur import Jp2k @@ -66,9 +68,13 @@ class TestUUIDXMP(unittest.TestCase): actual_ids = [b.box_id for b in jp2.box] self.assertEqual(actual_ids, expected_ids) - # The data should be an XMP packet, which gets interpreted as - # an ElementTree. - self.assertTrue(isinstance(jp2.box[-1].data, XMPMeta)) + # The data should be an XMP packet + if HAS_PYTHON_XMP_TOOLKIT: + # when python xmp toolkit is available. + self.assertTrue(isinstance(jp2.box[-1].data, XMPMeta)) + else: + # when python xmp toolkit is not available. + self.assertTrue(isinstance(jp2.box[-1].data, str)) class TestUUIDExif(unittest.TestCase): """Tests for UUIDs of Exif type.""" diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index c055aba..cf5596e 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -30,12 +30,13 @@ import warnings import numpy as np import pkg_resources -import libxmp - import glymur from glymur import Jp2k -from .fixtures import OPENJP2_IS_V2_OFFICIAL +from .fixtures import HAS_PYTHON_XMP_TOOLKIT, OPENJP2_IS_V2_OFFICIAL +if HAS_PYTHON_XMP_TOOLKIT: + import libxmp + from .fixtures import OPJ_DATA_ROOT, opj_data_file @@ -361,6 +362,8 @@ class TestJp2k(unittest.TestCase): self.assertEqual(ET.tostring(jp2k.box[3].xml.getroot()), b'this is a test') + @unittest.skipIf(not HAS_PYTHON_XMP_TOOLKIT, + "Requires Python XMP Toolkit >= 2.0") def test_xmp_attribute(self): """Verify the XMP packet in the shipping example file can be read.""" j = Jp2k(self.jp2file) From c8f6aff4d82c8782af745c8f4b5182aae597387c Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 24 Jan 2014 15:56:50 -0500 Subject: [PATCH 14/83] pylint work. #104 --- glymur/_uuid_io/Exif.py | 3 +- glymur/codestream.py | 4 +- glymur/jp2box.py | 29 ++++---- glymur/jp2k.py | 12 ++-- glymur/lib/openjpeg.py | 124 ++++++++++++++++---------------- glymur/lib/test/test_openjp2.py | 16 ++--- glymur/test/fixtures.py | 3 +- glymur/test/test_jp2box.py | 12 ++-- glymur/version.py | 16 +++-- 9 files changed, 111 insertions(+), 108 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index a7c04aa..cc8254e 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -18,7 +18,8 @@ else: # if the version is at least 2.0.0. try: import libxmp - if hasattr(libxmp, 'version') and re.match('[2-9].\d*.\d*', libxmp.version.VERSION): + if hasattr(libxmp, 'version') and re.match(r"""[2-9].\d*.\d*""", + libxmp.version.VERSION): from libxmp import XMPMeta _HAS_PYTHON_XMP_TOOLKIT = True else: diff --git a/glymur/codestream.py b/glymur/codestream.py index d01795f..67b2d56 100644 --- a/glymur/codestream.py +++ b/glymur/codestream.py @@ -663,7 +663,7 @@ class Codestream(object): bitdepth = tuple(((x & 0x7f) + 1) for x in data[0::3]) signed = tuple(((x & 0xb0) > 0) for x in data[0::3]) - + xrsiz = data[1::3] yrsiz = data[2::3] @@ -1529,7 +1529,7 @@ class SIZsegment(Segment): signed=self.signed, xyrsiz=(self.xrsiz, self.yrsiz)) return msg - + def __str__(self): msg = Segment.__str__(self) msg += '\n ' diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 8b0f6f3..85555e4 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -19,7 +19,6 @@ import os import pprint import struct import sys -import traceback import uuid import warnings import xml.etree.cElementTree as ET @@ -563,11 +562,11 @@ class CodestreamHeaderBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='jpch', longname='Codestream Header') self.length = length self.offset = offset - self.box = box + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.CodestreamHeaderBox(box={0})".format(self.box) @@ -625,12 +624,12 @@ class CompositingLayerHeaderBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='jplh', longname='Compositing Layer Header') self.length = length self.offset = offset - self.box = [] + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.CompositingLayerHeaderBox(box={0})" @@ -1073,11 +1072,11 @@ class AssociationBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='asoc', longname='Association') self.length = length self.offset = offset - self.box = box + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.AssociationBox(box={0})".format(self.box) @@ -1135,11 +1134,11 @@ class JP2HeaderBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='jp2h', longname='JP2 Header') self.length = length self.offset = offset - self.box = box + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.JP2HeaderBox(box={0})".format(self.box) @@ -1621,11 +1620,11 @@ class ResolutionBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='res ', longname='Resolution') self.length = length self.offset = offset - self.box = box + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.ResolutionBox(box={0})" @@ -2040,11 +2039,11 @@ class UUIDInfoBox(Jp2kBox): box : list List of boxes contained in this superbox. """ - def __init__(self, box=[], length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='uinf', longname='UUIDInfo') self.length = length self.offset = offset - self.box = box + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.UUIDInfoBox(box={0})".format(self.box) @@ -2214,8 +2213,8 @@ class UUIDBox(Jp2kBox): try: self._parse_raw_data() - except Exception as e: - warnings.warn(str(e)) + except RuntimeError as error: + warnings.warn(str(error)) def _parse_raw_data(self): """ diff --git a/glymur/jp2k.py b/glymur/jp2k.py index a337ae7..4eb8b2a 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -484,17 +484,17 @@ class Jp2k(Jp2kBox): stack.callback(opj2.image_destroy, image) _populate_image_struct(cparams, image, img_array) - + codec = opj2.create_compress(cparams.codec_fmt) stack.callback(opj2.destroy_codec, codec) - + info_handler = _INFO_CALLBACK if verbose else None opj2.set_info_handler(codec, info_handler) opj2.set_warning_handler(codec, _WARNING_CALLBACK) opj2.set_error_handler(codec, _ERROR_CALLBACK) - + opj2.setup_encoder(codec, cparams, image) - + if _OPENJP2_IS_OFFICIAL_V2: fptr = libc.fopen(self.filename, 'wb') strm = opj2.stream_create_default_file_stream(fptr, False) @@ -505,11 +505,11 @@ class Jp2k(Jp2kBox): strm = opj2.stream_create_default_file_stream_v3(self.filename, False) stack.callback(opj2.stream_destroy_v3, strm) - + opj2.start_compress(codec, image, strm) opj2.encode(codec, strm) opj2.end_compress(codec, strm) - + # Refresh the metadata. self.parse() diff --git a/glymur/lib/openjpeg.py b/glymur/lib/openjpeg.py index 5b1183f..418e9df 100644 --- a/glymur/lib/openjpeg.py +++ b/glymur/lib/openjpeg.py @@ -11,8 +11,8 @@ import numpy as np from .config import glymur_config _, OPENJPEG = glymur_config() -# Maximum number of tile parts expected by JPWL: increase at your will -JPWL_MAX_NO_TILESPECS = 16 +# Maximum number of tile parts expected by JPWL: increase at your will +JPWL_MAX_NO_TILESPECS = 16 J2K_MAXRLVLS = 33 # Number of maximum resolution level authorized PATH_LEN = 4096 # maximum allowed size for filenames @@ -58,7 +58,7 @@ class CommonStructType(ctypes.Structure): ("mj2_handle", ctypes.c_void_p)] -STREAM_READ = 0x0001 # The stream was opened for reading. +STREAM_READ = 0x0001 # The stream was opened for reading. STREAM_WRITE = 0x0002 # The stream was opened for writing. class CioType(ctypes.Structure): """Byte input-output stream (CIO) @@ -81,7 +81,7 @@ class CioType(ctypes.Structure): class CompressionInfoType(CommonStructType): - """Common fields between JPEG-2000 compression and decompression contexts. + """Common fields between JPEG-2000 compression and decompression contexts. This is for compression contexts. Corresponds to common_struct_t. """ pass @@ -91,68 +91,68 @@ class PocType(ctypes.Structure): """Progression order changes.""" _fields_ = [("resno", ctypes.c_int), # Resolution num start, Component num start, given by POC - ("compno0", ctypes.c_int), + ("compno0", ctypes.c_int), # Layer num end,Resolution num end, Component num end, given by POC - ("layno1", ctypes.c_int), - ("resno1", ctypes.c_int), - ("compno1", ctypes.c_int), + ("layno1", ctypes.c_int), + ("resno1", ctypes.c_int), + ("compno1", ctypes.c_int), - # Layer num start,Precinct num start, Precinct num end - ("layno0", ctypes.c_int), - ("precno0", ctypes.c_int), - ("precno1", ctypes.c_int), + # Layer num start,Precinct num start, Precinct num end + ("layno0", ctypes.c_int), + ("precno0", ctypes.c_int), + ("precno1", ctypes.c_int), # Progression order enum # OPJ_PROG_ORDER prg1,prg; - ("prg1", ctypes.c_int), - ("prg", ctypes.c_int), + ("prg1", ctypes.c_int), + ("prg", ctypes.c_int), - # Progression order string + # Progression order string # char progorder[5]; ("progorder", ctypes.c_char * 5), - # Tile number + # Tile number # int tile; - ("tile", ctypes.c_int), + ("tile", ctypes.c_int), # /** Start and end values for Tile width and height*/ # int tx0,tx1,ty0,ty1; - ("tx0", ctypes.c_int), - ("tx1", ctypes.c_int), - ("ty0", ctypes.c_int), - ("ty1", ctypes.c_int), + ("tx0", ctypes.c_int), + ("tx1", ctypes.c_int), + ("ty0", ctypes.c_int), + ("ty1", ctypes.c_int), # /** Start value, initialised in pi_initialise_encode*/ # int layS, resS, compS, prcS; - ("layS", ctypes.c_int), - ("resS", ctypes.c_int), - ("compS", ctypes.c_int), + ("layS", ctypes.c_int), + ("resS", ctypes.c_int), + ("compS", ctypes.c_int), ("prcS", ctypes.c_int), # /** End value, initialised in pi_initialise_encode */ # int layE, resE, compE, prcE; - ("layE", ctypes.c_int), - ("resE", ctypes.c_int), - ("compE", ctypes.c_int), - ("prcE", ctypes.c_int), + ("layE", ctypes.c_int), + ("resE", ctypes.c_int), + ("compE", ctypes.c_int), + ("prcE", ctypes.c_int), # Start and end values of Tile width and height, initialised in # pi_initialise_encode int txS,txE,tyS,tyE,dx,dy; - ("txS", ctypes.c_int), - ("txE", ctypes.c_int), - ("tyS", ctypes.c_int), - ("tyE", ctypes.c_int), - ("dx", ctypes.c_int), - ("dy", ctypes.c_int), + ("txS", ctypes.c_int), + ("txE", ctypes.c_int), + ("tyS", ctypes.c_int), + ("tyE", ctypes.c_int), + ("dx", ctypes.c_int), + ("dy", ctypes.c_int), - # Temporary values for Tile parts, initialised in pi_create_encode + # Temporary values for Tile parts, initialised in pi_create_encode # int lay_t, res_t, comp_t, prc_t,tx0_t,ty0_t; - ("lay_t", ctypes.c_int), - ("res_t", ctypes.c_int), - ("comp_t", ctypes.c_int), - ("prc_t", ctypes.c_int), - ("tx0_t", ctypes.c_int), + ("lay_t", ctypes.c_int), + ("res_t", ctypes.c_int), + ("comp_t", ctypes.c_int), + ("prc_t", ctypes.c_int), + ("tx0_t", ctypes.c_int), ("ty0_t", ctypes.c_int)] @@ -374,23 +374,23 @@ class DecompressionParametersType(ctypes.Structure): class ImageComptParmType(ctypes.Structure): """Component parameters structure used by the opj_image_create function. """ - _fields_ = [ - # XRsiz: horizontal separation of a sample of ith component with - # respect to the reference grid - ("dx", ctypes.c_int), + _fields_ = [ + # XRsiz: horizontal separation of a sample of ith component with + # respect to the reference grid + ("dx", ctypes.c_int), - # YRsiz: vertical separation of a sample of ith component with + # YRsiz: vertical separation of a sample of ith component with # respect to the reference grid */ - ("dy", ctypes.c_int), - - # data width, height - ("w", ctypes.c_int), - ("h", ctypes.c_int), + ("dy", ctypes.c_int), - # x component offset compared to the whole image - # y component offset compared to the whole image - ("x0", ctypes.c_int), - ("y0", ctypes.c_int), + # data width, height + ("w", ctypes.c_int), + ("h", ctypes.c_int), + + # x component offset compared to the whole image + # y component offset compared to the whole image + ("x0", ctypes.c_int), + ("y0", ctypes.c_int), # precision ('prec', ctypes.c_int), @@ -398,7 +398,7 @@ class ImageComptParmType(ctypes.Structure): # image depth in bits ('bpp', ctypes.c_int), - # signed (1) / unsigned (0) + # signed (1) / unsigned (0) ('sgnd', ctypes.c_int)] @@ -511,7 +511,7 @@ def destroy_compress(cinfo): def encode(cinfo, cio, image): """Wrapper for openjpeg library function opj_encode. - Encodes an image into a JPEG-2000 codestream. + Encodes an image into a JPEG-2000 codestream. Parameters ---------- @@ -540,7 +540,7 @@ def destroy_decompress(dinfo): def image_cmptparm_t_from_np(np_image): """Return appropriate image_cmptparm_t based on given numpy array. """ - try: + try: num_comps = np_image.shape[2] except IndexError: num_comps = 1 @@ -557,17 +557,17 @@ def image_cmptparm_t_from_np(np_image): bpp = 8 sgnd = 1 elif np_image.dtype == np.uint16: - prec = 16 - bpp = 16 + prec = 16 + bpp = 16 sgnd = 0 elif np_image.dtype == np.int16: - prec = 16 - bpp = 16 + prec = 16 + bpp = 16 sgnd = 1 else: raise(TypeError("unhandled")) - for j in range(0, num_comps): + for j in range(0, num_comps): tarr[j].dx = 1 tarr[j].dy = 1 tarr[j].w = np_image.shape[1] diff --git a/glymur/lib/test/test_openjp2.py b/glymur/lib/test/test_openjp2.py index 54d8254..8694507 100644 --- a/glymur/lib/test/test_openjp2.py +++ b/glymur/lib/test/test_openjp2.py @@ -1,5 +1,5 @@ """ -Tests for libopenjp2 wrapping functions. +Tests for libopenjp2 wrapping functions. """ # R0904: Seems like pylint is fooled in this situation # W0142: using kwargs is ok in this context @@ -212,7 +212,7 @@ class TestOpenJP2(unittest.TestCase): """Runs test designated tte3 in OpenJPEG test suite.""" with tempfile.NamedTemporaryFile(suffix=".j2k") as tfile: xtx3_setup(tfile.name) - self.assertTrue(True) + self.assertTrue(True) def test_rta3(self): """Runs test designated rta3 in OpenJPEG test suite.""" @@ -221,13 +221,13 @@ class TestOpenJP2(unittest.TestCase): codec_format = openjp2.CODEC_J2K self.j2k_random_tile_access(tfile.name, codec_format) - self.assertTrue(True) + self.assertTrue(True) def test_tte4(self): """Runs test designated tte4 in OpenJPEG test suite.""" with tempfile.NamedTemporaryFile(suffix=".j2k") as tfile: xtx4_setup(tfile.name) - self.assertTrue(True) + self.assertTrue(True) def test_rta4(self): """Runs test designated rta4 in OpenJPEG test suite.""" @@ -241,7 +241,7 @@ class TestOpenJP2(unittest.TestCase): """Runs test designated tte5 in OpenJPEG test suite.""" with tempfile.NamedTemporaryFile(suffix=".j2k") as tfile: xtx5_setup(tfile.name) - self.assertTrue(True) + self.assertTrue(True) def test_rta5(self): """Runs test designated rta5 in OpenJPEG test suite.""" @@ -332,8 +332,8 @@ def tile_encoder(**kwargs): def tile_decoder(**kwargs): """Fixture called with various configurations by many tests. - - Reads a tile. That's all it does. + + Reads a tile. That's all it does. """ stream = openjp2.stream_create_default_file_stream_v3(kwargs['filename'], True) @@ -355,7 +355,7 @@ def tile_decoder(**kwargs): openjp2.setup_decoder(codec, dparam) image = openjp2.read_header(stream, codec) - openjp2.set_decode_area(codec, image, + openjp2.set_decode_area(codec, image, kwargs['x0'], kwargs['y0'], kwargs['x1'], kwargs['y1']) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 164f93d..6626075 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -15,7 +15,8 @@ import glymur # if the version is at least 2.0.0. try: import libxmp - if hasattr(libxmp, 'version') and re.match('[2-9].\d*.\d*', libxmp.version.VERSION): + if hasattr(libxmp, 'version') and re.match(r'''[2-9].\d*.\d*''', + libxmp.version.VERSION): from libxmp import XMPMeta HAS_PYTHON_XMP_TOOLKIT = True else: diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index b83f86d..f731ff7 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -894,9 +894,9 @@ class TestRepr(unittest.TestCase): tree = ET.ElementTree(elt) box = glymur.jp2box.XMLBox(xml=tree) - regexp = "glymur.jp2box.XMLBox" - regexp += "\(xml=<(xml.etree.ElementTree.){0,1}ElementTree object " - regexp += "at 0x([a-f0-9]*)>\)" + regexp = r"""glymur.jp2box.XMLBox""" + regexp += r"""\(xml=<(xml.etree.ElementTree.){0,1}ElementTree object """ + regexp += """at 0x([a-f0-9]*)>\)""" if sys.hexversion < 0x03000000: self.assertRegexpMatches(repr(box), regexp) @@ -927,9 +927,9 @@ class TestRepr(unittest.TestCase): # Since the raw_data parameter is a sequence of bytes which could be # quite long, don't bother trying to make it conform to eval(repr()). - regexp = "glymur.jp2box.UUIDBox\(" - regexp += "the_uuid=UUID\('00000000-0000-0000-0000-000000000000'\),\s" - regexp += "raw_data=\)" + regexp = r"""glymur.jp2box.UUIDBox\(""" + regexp += """the_uuid=UUID\('00000000-0000-0000-0000-000000000000'\),\s""" + regexp += """raw_data=\)""" if sys.hexversion < 0x03000000: self.assertRegexpMatches(repr(box), regexp) diff --git a/glymur/version.py b/glymur/version.py index af7523e..260f880 100644 --- a/glymur/version.py +++ b/glymur/version.py @@ -1,10 +1,12 @@ -# This file is part of glymur, a Python interface for accessing JPEG 2000. -# -# http://glymur.readthedocs.org -# -# Copyright 2013 John Evans -# -# License: MIT +""" +This file is part of glymur, a Python interface for accessing JPEG 2000. + +http://glymur.readthedocs.org + +Copyright 2013 John Evans + +License: MIT +""" import sys import numpy as np From 77b58c1c5f956666816d9b40de746442c102c765 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 25 Jan 2014 15:36:55 -0500 Subject: [PATCH 15/83] No explicit dependence on libxmp anymore. Back to ElementTree. --- glymur/_uuid_io/Exif.py | 14 ++++++++++++++ glymur/_uuid_io/__init__.py | 2 +- glymur/jp2box.py | 28 ++++++++++++---------------- glymur/test/test_jp2box_uuid.py | 7 ++++--- glymur/test/test_jp2k.py | 5 ++++- glymur/test/test_printing.py | 12 ++++-------- 6 files changed, 39 insertions(+), 29 deletions(-) diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io/Exif.py index c1b30f0..967468d 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io/Exif.py @@ -6,6 +6,7 @@ import pprint import struct import sys import warnings +import xml.etree.cElementTree as ET if sys.hexversion < 0x02070000: # pylint: disable=F0401,E0611 @@ -13,8 +14,21 @@ if sys.hexversion < 0x02070000: else: from collections import OrderedDict +def xml(raw_data): + """ + XMP data to be parsed as XML. + """ + if sys.hexversion < 0x03000000: + elt = ET.fromstring(raw_data) + else: + text = raw_data.decode('utf-8') + elt = ET.fromstring(text) + + return ET.ElementTree(elt) + def tiff_header(read_buffer): """ + Interpret the uuid raw data as a tiff header. """ # Ignore the first six bytes. # Next 8 should be (73, 73, 42, 8) or (77, 77, 42, 8) diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py index d12d20c..778f912 100644 --- a/glymur/_uuid_io/__init__.py +++ b/glymur/_uuid_io/__init__.py @@ -1,4 +1,4 @@ """ Sub package for handling various types of UUIDs. """ -from .Exif import tiff_header +from .Exif import tiff_header, xml diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 300ac12..da8e1c9 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -33,12 +33,6 @@ else: import numpy as np -try: - from libxmp import XMPMeta - _HAS_PYTHON_XMP_TOOLKIT = True -except ImportError: - _HAS_PYTHON_XMP_TOOLKIT = False - from .codestream import Codestream from .core import _COLORSPACE_MAP_DISPLAY from .core import _COLOR_TYPE_MAP_DISPLAY @@ -2220,18 +2214,15 @@ class UUIDBox(Jp2kBox): try: self._parse_raw_data() - except Exception as e: - warnings.warn(str(e)) + except RuntimeError as error: + warnings.warn(str(error)) def _parse_raw_data(self): """ Private function for parsing UUID payloads if possible. """ if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - xmp = XMPMeta() - xmp.parse_from_str(self.raw_data.decode('utf-8'), - xmpmeta_wrap=False) - self.data = xmp + self.data = _uuid_io.xml(self.raw_data) elif self.uuid.bytes == b'JpgTiffExif->JP2': self.data = _uuid_io.tiff_header(self.raw_data) else: @@ -2243,11 +2234,16 @@ class UUIDBox(Jp2kBox): return msg.format(repr(self.uuid), len(self.data)) def __str__(self): - msg = '{0}\n' - msg += ' UUID: {1}\n' - msg += ' UUID Data: {2}' + msg = '{0}\n UUID: {1}'.format(Jp2kBox.__str__(self), self.uuid) - msg = msg.format(Jp2kBox.__str__(self), self.uuid, str(self.data)) + if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): + line = ' (XMP)\n UUID Data: {0}' + msg += line.format(_pretty_print_xml(self.data)) + elif self.uuid.bytes == b'JpgTiffExif->JP2': + msg += ' (EXIF)\n UUID Data: {0}'.format(str(self.data)) + else: + line = ' (unknown)\n UUID Data: {0} bytes' + msg += line.format(len(self.raw_data)) return msg diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index 8f24f74..0d3684a 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -18,7 +18,7 @@ import sys import tempfile import uuid import warnings -from xml.etree import cElementTree as ET +import xml.etree if sys.hexversion < 0x02070000: import unittest2 as unittest @@ -68,7 +68,8 @@ class TestUUIDXMP(unittest.TestCase): # The data should be an XMP packet, which gets interpreted as # an ElementTree. - self.assertTrue(isinstance(jp2.box[-1].data, XMPMeta)) + self.assertTrue(isinstance(jp2.box[-1].data, + xml.etree.ElementTree.ElementTree)) class TestUUIDExif(unittest.TestCase): """Tests for UUIDs of Exif type.""" @@ -184,7 +185,7 @@ class TestUUIDExif(unittest.TestCase): tfile.flush() jp2 = glymur.Jp2k(tfile.name) - self.assertEqual(jp2.box[-1].data.ifds['Image']['Make'], "HTC") + self.assertEqual(jp2.box[-1].data['Make'], "HTC") if __name__ == "__main__": unittest.main() diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index c055aba..aeb0e41 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -31,6 +31,7 @@ import numpy as np import pkg_resources import libxmp +from libxmp import XMPMeta import glymur from glymur import Jp2k @@ -364,7 +365,9 @@ class TestJp2k(unittest.TestCase): def test_xmp_attribute(self): """Verify the XMP packet in the shipping example file can be read.""" j = Jp2k(self.jp2file) - xmp = j.box[3].data + xmp = XMPMeta() + xmp.parse_from_str(j.box[3].raw_data.decode('utf-8'), + xmpmeta_wrap=False) creator_tool = xmp.get_property(libxmp.consts.XMP_NS_XMP, 'CreatorTool') self.assertEqual(creator_tool, 'Google') diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index a1f0c55..382e28d 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -1057,6 +1057,8 @@ class TestPrinting(unittest.TestCase): tfile.write(struct.pack(' Date: Sat, 25 Jan 2014 15:55:36 -0500 Subject: [PATCH 16/83] Fixing tests broken by bad merge. --- glymur/test/fixtures.py | 174 +++++++++++++++++------------------ glymur/test/test_printing.py | 1 - 2 files changed, 84 insertions(+), 91 deletions(-) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 6626075..e11709b 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -183,96 +183,90 @@ def read_pgx_header(pgx_file): nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - -""" + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + """ SimpleRDF = """ Date: Sat, 25 Jan 2014 16:04:57 -0500 Subject: [PATCH 17/83] Removed restriction on only writing XMP UUIDs. #104 --- glymur/jp2box.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index b3fcf89..6dbec1d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2249,9 +2249,6 @@ class UUIDBox(Jp2kBox): def write(self, fptr): """Write a UUID box to file. """ - if self.uuid != uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - msg = "Only XMP UUID boxes can currently be written." - raise NotImplementedError(msg) write_buffer = struct.pack('>I4s', self.length, b'uuid') fptr.write(write_buffer) fptr.write(self.uuid.bytes) From d6a8736aea1c8ed2469ba10235701fb5935f6f47 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 25 Jan 2014 17:07:33 -0500 Subject: [PATCH 18/83] Removed shutil debugging statement. --- glymur/test/test_printing.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 2828c96..b6e3484 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -1056,8 +1056,6 @@ class TestPrinting(unittest.TestCase): tfile.write(struct.pack(' Date: Sat, 25 Jan 2014 17:08:53 -0500 Subject: [PATCH 19/83] Removed python-xmp-toolkit requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b4fdc2f..b780e4d 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ kwargs = {'name': 'Glymur', 'license': 'MIT', 'test_suite': 'glymur.test'} -instllrqrs = ['numpy>=1.4.1', 'python-xmp-toolkit>=2.0.0'] +instllrqrs = ['numpy>=1.4.1'] if sys.hexversion < 0x03030000: instllrqrs.append('contextlib2>=0.4') instllrqrs.append('mock>=1.0.1') From 3efd4169d947ec817a0a997d7fb06413012eed78 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 25 Jan 2014 18:14:31 -0500 Subject: [PATCH 20/83] Added fixtures for ICC profiles for 27, 33, and 34. --- glymur/test/fixtures.py | 67 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index e11709b..ff33237 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -283,3 +283,70 @@ SimpleRDF = """ """ + +text_gbr_27 = """Colour Specification Box (colr) @ (179, 1339) + Method: any ICC profile + Precedence: 2 + Approximation: accurately represents correct colorspace definition + ICC Profile: + {'Color Space': 'RGB', + 'Connection Space': 'XYZ', + 'Creator': u'appl', + 'Datetime': datetime.datetime(2009, 2, 25, 11, 26, 11), + 'Device Attributes': 'reflective, glossy, positive media polarity, color media', + 'Device Class': 'display device profile', + 'Device Manufacturer': u'appl', + 'Device Model': '', + 'File Signature': u'acsp', + 'Flags': 'not embedded, can be used independently', + 'Illuminant': array([ 0.96420288, 1. , 0.8249054 ]), + 'Platform': u'APPL', + 'Preferred CMM Type': 1634758764, + 'Rendering Intent': 'perceptual', + 'Size': 1328, + 'Version': '2.2.0'}""" + +text_gbr_33 = """Colour Specification Box (colr) @ (179, 1339) + Method: any ICC profile + Precedence: 2 + Approximation: accurately represents correct colorspace definition + ICC Profile: + {'Size': 1328, + 'Preferred CMM Type': 1634758764, + 'Version': '2.2.0', + 'Device Class': 'display device profile', + 'Color Space': 'RGB', + 'Connection Space': 'XYZ', + 'Datetime': datetime.datetime(2009, 2, 25, 11, 26, 11), + 'File Signature': 'acsp', + 'Platform': 'APPL', + 'Flags': 'not embedded, can be used independently', + 'Device Manufacturer': 'appl', + 'Device Model': '', + 'Device Attributes': 'reflective, glossy, positive media polarity, color media', + 'Rendering Intent': 'perceptual', + 'Illuminant': array([ 0.96420288, 1. , 0.8249054 ]), + 'Creator': 'appl'}""" + +text_gbr_34 = """Colour Specification Box (colr) @ (179, 1339) + Method: any ICC profile + Precedence: 2 + Approximation: accurately represents correct colorspace definition + ICC Profile: + {'Size': 1328, + 'Preferred CMM Type': 1634758764, + 'Version': '2.2.0', + 'Device Class': 'display device profile', + 'Color Space': 'RGB', + 'Connection Space': 'XYZ', + 'Datetime': datetime.datetime(2009, 2, 25, 11, 26, 11), + 'File Signature': 'acsp', + 'Platform': 'APPL', + 'Flags': 'not embedded, can be used independently', + 'Device Manufacturer': 'appl', + 'Device Model': '', + 'Device Attributes': 'reflective, glossy, positive media polarity, color ' + 'media', + 'Rendering Intent': 'perceptual', + 'Illuminant': array([ 0.96420288, 1. , 0.8249054 ]), + 'Creator': 'appl'}""" From 8617ce04421e5f44da83b3f43c2c97dea1af6b49 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 25 Jan 2014 18:16:54 -0500 Subject: [PATCH 21/83] Using fixtures for expected values. Removed one duplicated test. #104 --- glymur/test/test_printing.py | 109 +++-------------------------------- 1 file changed, 9 insertions(+), 100 deletions(-) diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index b6e3484..892e283 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -36,6 +36,7 @@ else: import glymur from glymur import Jp2k from .fixtures import OPJ_DATA_ROOT, opj_data_file, nemo_xmp_box +from .fixtures import text_gbr_27, text_gbr_33, text_gbr_34 @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") @@ -284,75 +285,6 @@ class TestPrinting(unittest.TestCase): expected = '\n'.join(lines) self.assertEqual(actual, expected) - @unittest.skipIf(OPJ_DATA_ROOT is None, - "OPJ_DATA_ROOT environment variable not set") - def test_icc_profile(self): - """verify printing of colr box with ICC profile""" - filename = opj_data_file('input/nonregression/text_GBR.jp2') - with warnings.catch_warnings(): - # brand is 'jp2 ', but has any icc profile. - warnings.simplefilter("ignore") - jp2 = Jp2k(filename) - with patch('sys.stdout', new=StringIO()) as fake_out: - print(jp2.box[3].box[1]) - actual = fake_out.getvalue().strip() - lin27 = ["Colour Specification Box (colr) @ (179, 1339)", - " Method: any ICC profile", - " Precedence: 2", - " Approximation: accurately represents correct " - + "colorspace definition", - " ICC Profile:", - " {'Color Space': 'RGB',", - " 'Connection Space': 'XYZ',", - " 'Creator': u'appl',", - " 'Datetime': " - + "datetime.datetime(2009, 2, 25, 11, 26, 11),", - " 'Device Attributes': 'reflective, glossy, " - + "positive media polarity, color media',", - " 'Device Class': 'display device profile',", - " 'Device Manufacturer': u'appl',", - " 'Device Model': '',", - " 'File Signature': u'acsp',", - " 'Flags': " - + "'not embedded, can be used independently',", - " 'Illuminant': " - + "array([ 0.96420288, 1. , 0.8249054 ]),", - " 'Platform': u'APPL',", - " 'Preferred CMM Type': 1634758764,", - " 'Rendering Intent': 'perceptual',", - " 'Size': 1328,", - " 'Version': '2.2.0'}"] - lin33 = ["Colour Specification Box (colr) @ (179, 1339)", - " Method: any ICC profile", - " Precedence: 2", - " Approximation: accurately represents correct " - + "colorspace definition", - " ICC Profile:", - " {'Size': 1328,", - " 'Preferred CMM Type': 1634758764,", - " 'Version': '2.2.0',", - " 'Device Class': 'display device profile',", - " 'Color Space': 'RGB',", - " 'Connection Space': 'XYZ',", - " 'Datetime': " - + "datetime.datetime(2009, 2, 25, 11, 26, 11),", - " 'File Signature': 'acsp',", - " 'Platform': 'APPL',", - " 'Flags': 'not embedded, can be used " - + "independently',", - " 'Device Manufacturer': 'appl',", - " 'Device Model': '',", - " 'Device Attributes': 'reflective, glossy, " - + "positive media polarity, color media',", - " 'Rendering Intent': 'perceptual',", - " 'Illuminant': " - + "array([ 0.96420288, 1. , 0.8249054 ]),", - " 'Creator': 'appl'}"] - - lines = lin27 if sys.hexversion < 0x03000000 else lin33 - expected = '\n'.join(lines) - self.assertEqual(actual, expected) - @unittest.skipIf(OPJ_DATA_ROOT is None, "OPJ_DATA_ROOT environment variable not set") def test_crg(self): @@ -963,12 +895,10 @@ class TestPrinting(unittest.TestCase): expected = '\n'.join(lines) self.assertEqual(actual, expected) - @unittest.skipIf(sys.hexversion < 0x03000000, - "Ordered dicts not printing well in 2.7") @unittest.skipIf(OPJ_DATA_ROOT is None, "OPJ_DATA_ROOT environment variable not set") - def test_jpx_approx_icc_profile(self): - """verify jpx with approx field equal to zero""" + def test_icc_profile(self): + """verify icc profile printing with a jpx""" # ICC profiles may be used in JP2, but the approximation field should # be zero unless we have jpx. This file does both. filename = opj_data_file('input/nonregression/text_GBR.jp2') @@ -980,34 +910,13 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(jp2.box[3].box[1]) actual = fake_out.getvalue().strip() - lines = ["Colour Specification Box (colr) @ (179, 1339)", - " Method: any ICC profile", - " Precedence: 2", - " Approximation: accurately represents " - + "correct colorspace definition", - " ICC Profile:", - " {'Size': 1328,", - " 'Preferred CMM Type': 1634758764,", - " 'Version': '2.2.0',", - " 'Device Class': 'display device profile',", - " 'Color Space': 'RGB',", - " 'Connection Space': 'XYZ',", - " 'Datetime': " - + "datetime.datetime(2009, 2, 25, 11, 26, 11),", - " 'File Signature': 'acsp',", - " 'Platform': 'APPL',", - " 'Flags': 'not embedded, " - + "can be used independently',", - " 'Device Manufacturer': 'appl',", - " 'Device Model': '',", - " 'Device Attributes': 'reflective, glossy, " - + "positive media polarity, color media',", - " 'Rendering Intent': 'perceptual',", - " 'Illuminant': array([ 0.96420288, 1. ," - + " 0.8249054 ]),", - " 'Creator': 'appl'}"] + if sys.hexversion < 0x03000000: + expected = text_gbr_27 + elif sys.hexversion < 0x03040000: + expected = text_gbr_33 + else: + expected = text_gbr_34 - expected = '\n'.join(lines) self.assertEqual(actual, expected) @unittest.skipIf(OPJ_DATA_ROOT is None, From 052cadbf8e1900009a322f4c6cab56e3f548fdcb Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 27 Jan 2014 16:48:40 -0500 Subject: [PATCH 22/83] No longer presenting ElementTree as viable for XMP in docs. #104 --- docs/source/how_do_i.rst | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index 443e8db..0fb4d9d 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -236,28 +236,21 @@ The example JP2 file shipped with glymur has an XMP UUID. :: xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" ns0:xmptk="Exempi + XMP Core 5.1.2"> + + Google + 2013-02-09T14:47:53 + + . . . Since the UUID data in this case is returned as an ElementTree instance, -one can use ElementTree from the standard library to access the data. -For example, to extract the **CreatorTool** attribute value, the following -would work:: - - >>> xmp = j.box[3].data.packet - >>> rdf = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' - >>> ns2 = '{http://ns.adobe.com/xap/1.0/}' - >>> name = '{0}RDF/{0}Description/{1}CreatorTool'.format(rdf, ns2) - >>> elt = xmp.find(name) - >>> elt - - >>> elt.text - 'Google' - -Yes, that's painful. A better solution is to install the Python XMP Toolkit -(developer branch):: +one might first turn to ElementTree from the standard library. There's +a better solution though, particularly if you need to create XMP, and that +is to Use the Python XMP Toolkit instead (make sure you use version 2.0 and +not 1.0.2).:: >>> from libxmp import XMPMeta >>> from libxmp.consts import XMP_NS_XMP as NS_XAP From f214459ef7fe0ab34686e0dd5467d2910c731cda Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 30 Jan 2014 18:48:56 -0500 Subject: [PATCH 23/83] Starting to fill out python xmp toolkit advocacy. #104 --- docs/source/how_do_i.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index 443e8db..e620dbc 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -243,8 +243,8 @@ The example JP2 file shipped with glymur has an XMP UUID. :: Since the UUID data in this case is returned as an ElementTree instance, one can use ElementTree from the standard library to access the data. -For example, to extract the **CreatorTool** attribute value, the following -would work:: +For example, to extract the **CreatorTool** attribute value, one could do the +following >>> xmp = j.box[3].data.packet >>> rdf = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' @@ -256,8 +256,8 @@ would work:: >>> elt.text 'Google' -Yes, that's painful. A better solution is to install the Python XMP Toolkit -(developer branch):: +But that would be painful. A better solution is to install the Python XMP +Toolkit:: >>> from libxmp import XMPMeta >>> from libxmp.consts import XMP_NS_XMP as NS_XAP @@ -266,3 +266,16 @@ Yes, that's painful. A better solution is to install the Python XMP Toolkit >>> meta.get_property(NS_XAP, 'CreatorTool') 'Google' +Where the Python XMP Toolkit can really shine, though, is when you are +converting an image from another format such as TIFF or JPEG into JPEG 2000. +For example:: + + >>> import requests + >>> r = requests.get('http://photojournal.jpl.nasa.gov/tiff/PIA17145.tif') + >>> with open('PIA17145.tif', 'wb') as fptr: fptr.write(r.content) + >>> from libxmp import XMPFiles + >>> xf = XMPFile() + >>> xf.open_file('PIA17145.tif') + >>> xmp = xf.get_xmp() + + From 6e9846ca9137e7625aaf09f09ae4f0dd45e32587 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 30 Jan 2014 18:49:48 -0500 Subject: [PATCH 24/83] Fixed repr for XMP uuids. --- glymur/jp2box.py | 2 +- glymur/test/test_jp2box.py | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 6dbec1d..122d5c4 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2230,7 +2230,7 @@ class UUIDBox(Jp2kBox): def __repr__(self): msg = "glymur.jp2box.UUIDBox(the_uuid={0}, " msg += "raw_data=)" - return msg.format(repr(self.uuid), len(self.data)) + return msg.format(repr(self.uuid), len(self.raw_data)) def __str__(self): msg = '{0}\n UUID: {1}'.format(Jp2kBox.__str__(self), self.uuid) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index f731ff7..a40a5c6 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -919,7 +919,7 @@ class TestRepr(unittest.TestCase): self.assertEqual(box.vendor_mask, newbox.vendor_mask) @unittest.skipIf(sys.hexversion < 0x02070000, "Requires 2.7+") - def test_uuid_box(self): + def test_uuid_box_generic(self): """Verify uuid repr method.""" uuid_instance = uuid.UUID('00000000-0000-0000-0000-000000000000') data = b'0123456789' @@ -936,6 +936,24 @@ class TestRepr(unittest.TestCase): else: self.assertRegex(repr(box), regexp) + @unittest.skipIf(sys.hexversion < 0x02070000, "Requires 2.7+") + def test_uuid_box_xmp(self): + """Verify uuid repr method for XMP UUID box.""" + jp2file = glymur.data.nemo() + j = Jp2k(jp2file) + box = j.box[3] + + # Since the raw_data parameter is a sequence of bytes which could be + # quite long, don't bother trying to make it conform to eval(repr()). + regexp = r"""glymur.jp2box.UUIDBox\(""" + regexp += """the_uuid=UUID\('be7acfcb-97a9-42e8-9c71-999491e3afac'\),\s""" + regexp += """raw_data=\)""" + + if sys.hexversion < 0x03000000: + self.assertRegexpMatches(repr(box), regexp) + else: + self.assertRegex(repr(box), regexp) + @unittest.skipIf(sys.hexversion < 0x02070000, "Requires 2.7+") def test_contiguous_codestream_box(self): """Verify contiguous codestream box repr method.""" From f3fca7d7d328d3e9e74911623ddd9de7872cdac0 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 1 Feb 2014 13:21:51 -0500 Subject: [PATCH 25/83] More xmp blurbs. --- docs/source/how_do_i.rst | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index e620dbc..3aaa300 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -268,14 +268,31 @@ Toolkit:: Where the Python XMP Toolkit can really shine, though, is when you are converting an image from another format such as TIFF or JPEG into JPEG 2000. -For example:: +For example, if you were to be converting the TIFF image found at +http://photojournal.jpl.nasa.gov/tiff/PIA17145.tif info JPEG 2000:: + + >>> import skimage.io + >>> image = skimage.io.imread('PIA17145.tif') + >>> from glymur import Jp2k + >>> jp2 = Jp2k('PIA17145.jp2', 'wb') + >>> jp2.write(image) + +Next you can extract the XMP metadata. - >>> import requests - >>> r = requests.get('http://photojournal.jpl.nasa.gov/tiff/PIA17145.tif') - >>> with open('PIA17145.tif', 'wb') as fptr: fptr.write(r.content) >>> from libxmp import XMPFiles >>> xf = XMPFile() >>> xf.open_file('PIA17145.tif') >>> xmp = xf.get_xmp() +If you are familiar with TIFF, you can verify that there's no XMP tag in the +TIFF file, but the Python XMP Toolkit takes advantage of the TIFF header +structure to populate an XMP packet for you. If you were working with a JPEG +file with Exif metadata, that information would be included in the XMP packet +as well. Now you can append the XMP packet in a UUIDBox:: + + >>> import uuid + >>> the_uuid = uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac') + >>> box = glymur.jp2box.UUIDBox(uuid, str(xmp).encode()) + >>> jp2.append(box) + From 3fc782b6fd17226705c86688ed252eda9c396915 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 6 Feb 2014 10:14:10 -0500 Subject: [PATCH 26/83] Refactored superbox writing. Closes #153. At the moment this only affects jp2s and asoc superboxes for now. --- glymur/jp2box.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 83d768d..ca1d973 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -88,6 +88,26 @@ class Jp2kBox(object): msg = "Not supported for {0} box.".format(self.longname) raise NotImplementedError(msg) + def _write_superbox(self, fptr): + """Write a superbox. + + Parameters + ---------- + fptr : file or file object + Superbox (box of boxes) to be written to this file. + """ + # Write the contained boxes, then come back and write the length. + orig_pos = fptr.tell() + fptr.write(struct.pack('>I', 0)) + fptr.write(self.box_id.encode()) + for box in self.box: + box.write(fptr) + + end_pos = fptr.tell() + fptr.seek(orig_pos) + fptr.write(struct.pack('>I', end_pos - orig_pos)) + fptr.seek(end_pos) + def parse_superbox(self, fptr): """Parse a superbox (box consisting of nothing but other boxes. @@ -1372,17 +1392,7 @@ class AssociationBox(Jp2kBox): def write(self, fptr): """Write an association box to file. """ - # Write the contained boxes, then come back and write the length. - orig_pos = fptr.tell() - fptr.write(struct.pack('>I', 0)) - fptr.write('asoc'.encode()) - for box in self.box: - box.write(fptr) - - end_pos = fptr.tell() - fptr.seek(orig_pos) - fptr.write(struct.pack('>I', end_pos - orig_pos)) - fptr.seek(end_pos) + self._write_superbox(fptr) class JP2HeaderBox(Jp2kBox): @@ -1424,17 +1434,7 @@ class JP2HeaderBox(Jp2kBox): def write(self, fptr): """Write a JP2 Header box to file. """ - # Write the contained boxes, then come back and write the length. - orig_pos = fptr.tell() - fptr.write(struct.pack('>I', 0)) - fptr.write('jp2h'.encode()) - for box in self.box: - box.write(fptr) - - end_pos = fptr.tell() - fptr.seek(orig_pos) - fptr.write(struct.pack('>I', end_pos - orig_pos)) - fptr.seek(end_pos) + self._write_superbox(fptr) @staticmethod def parse(fptr, offset, length): From 1e3556e9017394d5b187ccad493185b8a6b7620a Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 6 Feb 2014 20:45:26 -0500 Subject: [PATCH 27/83] Added write support for DataEntryURL box. Closes #154 --- CHANGES.txt | 15 ++++++++------- glymur/jp2box.py | 12 ++++++++++++ glymur/test/test_jp2box.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 0a8b277..25dc20c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,11 +1,12 @@ Feb 03, 2014 - Removed support for Python 2.6. Added write support for - JP2 Palette and Component Mapping boxes, JPX Association and - NumberList boxes. Added read support for JPX free, number list, - data reference, fragment table, and fragment list boxes. Palette - box now a 2D numpy array instead of a list of 1D arrays. JP2 - super box constructors now take optional box list argument. - Fixed bug where JPX files with more than one codestream but - advertising jp2 compatibility were not being read. + JP2 DataEntryURL, Palette and Component Mapping boxes, JPX + Association and NumberList boxes. Added read support for JPX + free, number list, data reference, fragment table, and fragment + list boxes. Palette box now a 2D numpy array instead of a list + of 1D arrays. JP2 super box constructors now take optional box + list argument. Fixed bug where JPX files with more than one + codestream but advertising jp2 compatibility were not being + read. Jan 28, 2014 - v0.5.10 Fixed bad warning when reader requirements box mask length is unsupported. diff --git a/glymur/jp2box.py b/glymur/jp2box.py index ca1d973..47cfa30 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2518,6 +2518,18 @@ class DataEntryURLBox(Jp2kBox): self.length = length self.offset = offset + def write(self, fptr): + """Write a data entry url box to file. + """ + length = 8 + 1 + 3 + len(self.url.encode()) + write_buffer = struct.pack('>I4sBBBB', + length, self.box_id.encode(), + self.version, + self.flag[0], self.flag[1], self.flag[2]) + fptr.write(write_buffer) + fptr.write(self.url.encode()) + + def __repr__(self): msg = "glymur.jp2box.DataEntryURLBox({0}, {1}, '{2}')" msg = msg.format(self.version, self.flag, self.url) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 0a3b24a..a4930a9 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -52,6 +52,34 @@ def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite('glymur.jp2box')) return tests +@unittest.skipIf(os.name == "nt", "Temporary file issue on window.") +class TestDataEntryURL(unittest.TestCase): + """Test suite for DataEntryURL boxes.""" + def setUp(self): + self.jp2file = glymur.data.nemo() + + def test_basic_url(self): + """Just your most basic URL box.""" + # Wrap our j2k file in a JP2 box along with an interior url box. + jp2 = Jp2k(self.jp2file) + + url = 'http://glymur.readthedocs.org' + deurl = glymur.jp2box.DataEntryURLBox(0, (0, 0, 0), url) + boxes = [box for box in jp2.box if box.box_id != 'uuid'] + boxes.append(deurl) + with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: + jp22 = jp2.wrap(tfile.name, boxes=boxes) + import shutil + shutil.copyfile(tfile.name, '/Users/jevans/a.jp2') + + actdata = [box.box_id for box in jp22.box] + expdata = ['jP ', 'ftyp', 'jp2h', 'jp2c', 'url '] + self.assertEqual(actdata, expdata) + self.assertEqual(jp22.box[4].version, 0) + self.assertEqual(jp22.box[4].flag, (0, 0, 0)) + self.assertEqual(jp22.box[4].url, url) + + @unittest.skipIf(glymur.version.openjpeg_version_tuple[0] < 2 or OPENJP2_IS_V2_OFFICIAL, "Not supported until 2.0+.") From 03ea0f4b54ec77eb8db6a6089ab7a79ea693ff2f Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 6 Feb 2014 21:11:42 -0500 Subject: [PATCH 28/83] Removed test artifact. Closes #154 --- glymur/test/test_jp2box.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index a4930a9..c647bd2 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -69,8 +69,6 @@ class TestDataEntryURL(unittest.TestCase): boxes.append(deurl) with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: jp22 = jp2.wrap(tfile.name, boxes=boxes) - import shutil - shutil.copyfile(tfile.name, '/Users/jevans/a.jp2') actdata = [box.box_id for box in jp22.box] expdata = ['jP ', 'ftyp', 'jp2h', 'jp2c', 'url '] From fcbc8c52b4ab3ba9af197918b04b5425a19cecc5 Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 7 Feb 2014 16:52:28 -0500 Subject: [PATCH 29/83] Added new error message for case when wrapping jp2h box is empty. Closes #155 --- glymur/jp2k.py | 6 ++++++ glymur/test/test_jp2box.py | 20 ++++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index ea99a44..732a79f 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1132,6 +1132,12 @@ def _validate_jp2_box_sequence(boxes): msg = "The codestream box must be preceeded by a jp2 header box." raise IOError(msg) + # 1st jp2 header box cannot be empty. + jp2h = boxes[jp2h_idx] + if len(jp2h.box) == 0: + msg = "The JP2 header superbox cannot be empty." + raise IOError(msg) + # 1st jp2 header box must be ihdr jp2h = boxes[jp2h_idx] if jp2h.box[0].box_id != 'ihdr': diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index c647bd2..831c4c6 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -510,6 +510,7 @@ class TestAppend(unittest.TestCase): jp2.append(uuidbox) +@unittest.skipIf(os.name == "nt", "Temporary file issue on window.") class TestWrap(unittest.TestCase): """Tests for wrap method.""" @@ -570,7 +571,6 @@ class TestWrap(unittest.TestCase): self.assertEqual(jp2.box[2].box[1].colorspace, glymur.core.SRGB) self.assertIsNone(jp2.box[2].box[1].icc_profile) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_wrap(self): """basic test for rewrapping a j2c file, no specified boxes""" j2k = Jp2k(self.j2kfile) @@ -578,7 +578,6 @@ class TestWrap(unittest.TestCase): j2k.wrap(tfile.name) self.verify_wrapped_raw(tfile.name) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_jpx_to_jp2(self): """basic test for rewrapping a jpx file""" with warnings.catch_warnings(): @@ -602,7 +601,6 @@ class TestWrap(unittest.TestCase): for j, offset in enumerate(expected_offsets): self.assertEqual(jp2.box[j].offset, offset) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_wrap_jp2(self): """basic test for rewrapping a jp2 file, no specified boxes""" j2k = Jp2k(self.jp2file) @@ -611,7 +609,17 @@ class TestWrap(unittest.TestCase): boxes = [box.box_id for box in jp2.box] self.assertEqual(boxes, ['jP ', 'ftyp', 'jp2h', 'jp2c']) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") + def test_empty_jp2h(self): + """JP2H box list cannot be empty.""" + jp2 = Jp2k(self.jp2file) + with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: + boxes = jp2.box + # Right here the jp2h superbox has two child boxes. Empty out that + # list to trigger the error. + boxes[2].box = [] + with self.assertRaises(IOError): + jp22 = jp2.wrap(tfile.name, boxes=boxes) + def test_default_layout_with_boxes(self): """basic test for rewrapping a jp2 file, boxes specified""" j2k = Jp2k(self.j2kfile) @@ -631,7 +639,6 @@ class TestWrap(unittest.TestCase): j2k.wrap(tfile.name, boxes=boxes) self.verify_wrapped_raw(tfile.name) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_ihdr_not_first_in_jp2h(self): """The specification says that ihdr must be the first box in jp2h.""" j2k = Jp2k(self.j2kfile) @@ -651,7 +658,6 @@ class TestWrap(unittest.TestCase): with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_first_boxes_jp_and_ftyp(self): """first two boxes must be jP followed by ftyp""" j2k = Jp2k(self.j2kfile) @@ -673,7 +679,6 @@ class TestWrap(unittest.TestCase): with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_jp2h_not_preceeding_jp2c(self): """jp2h must precede jp2c""" j2k = Jp2k(self.j2kfile) @@ -695,7 +700,6 @@ class TestWrap(unittest.TestCase): with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) - @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") def test_missing_codestream(self): """Need a codestream box in order to call wrap method.""" j2k = Jp2k(self.j2kfile) From 14743d6633d02c60384816981db6ced0828a72e3 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 8 Feb 2014 14:56:50 -0500 Subject: [PATCH 30/83] Added write support for dtbl boxes. --- CHANGES.txt | 16 ++++---- glymur/jp2box.py | 21 ++++++++++ glymur/jp2k.py | 52 +++++++++++++++++++++++ glymur/test/test_jp2box_jpx.py | 75 +++++++++++++++++++++------------- 4 files changed, 128 insertions(+), 36 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 25dc20c..bc823bd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,12 +1,12 @@ -Feb 03, 2014 - Removed support for Python 2.6. Added write support for +Feb 08, 2014 - Removed support for Python 2.6. Added write support for JP2 DataEntryURL, Palette and Component Mapping boxes, JPX - Association and NumberList boxes. Added read support for JPX - free, number list, data reference, fragment table, and fragment - list boxes. Palette box now a 2D numpy array instead of a list - of 1D arrays. JP2 super box constructors now take optional box - list argument. Fixed bug where JPX files with more than one - codestream but advertising jp2 compatibility were not being - read. + Association, NumberList and DataReference boxes. Added read + support for JPX free, number list, data reference, fragment + table, and fragment list boxes. Palette box now a 2D numpy + array instead of a list of 1D arrays. JP2 super box constructors + now take optional box list argument. Fixed bug where JPX files + with more than one codestream but advertising jp2 compatibility + were not being read. Jan 28, 2014 - v0.5.10 Fixed bad warning when reader requirements box mask length is unsupported. diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 47cfa30..d535af9 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -863,6 +863,27 @@ class DataReferenceBox(Jp2kBox): self.length = length self.offset = offset + def write(self, fptr): + """Write a Data Reference box to file. + """ + + # Very similar to the say a superbox is written. + orig_pos = fptr.tell() + fptr.write(struct.pack('>I', 0)) + fptr.write(self.box_id.encode()) + + # Write the number of data entry url boxes. + write_buffer = struct.pack('>H', len(self.DR)) + fptr.write(write_buffer) + + for box in self.DR: + box.write(fptr) + + end_pos = fptr.tell() + fptr.seek(orig_pos) + fptr.write(struct.pack('>I', end_pos - orig_pos)) + fptr.seek(end_pos) + def __str__(self): msg = Jp2kBox.__str__(self) for box in self.DR: diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 732a79f..f9e1030 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -16,6 +16,7 @@ if sys.hexversion >= 0x03030000: else: from contextlib2 import ExitStack +from collections import Counter import ctypes import math import os @@ -1190,6 +1191,57 @@ def _validate_jp2_box_sequence(boxes): _asoc_check(boxes) _jpx_brand(boxes, boxes[1].brand) _jpx_compatibility(boxes, boxes[1].compatibility_list) + _check_for_singletons(boxes) + _check_top_level(boxes) + +def _collect_box_count(boxes): + """Count the occurences of each box type.""" + count = Counter([box.box_id for box in boxes]) + + # Add the counts in the superboxes. + for box in boxes: + if hasattr(box, 'box'): + count.update(_collect_box_count(box.box)) + + return count + +TOP_LEVEL_ONLY_BOXES = set(['dtbl']) + +def _check_superbox_for_top_levels(boxes): + """Several boxes can only occur at the top level.""" + # We are only looking at the boxes contained in a superbox, so if any of + # the blacklisted boxes show up here, it's an error. + box_ids = set([box.box_id for box in boxes]) + intersection = box_ids.intersection(TOP_LEVEL_ONLY_BOXES) + if len(intersection) > 0: + msg = "A '{0}' box cannot be nested in a superbox." + raise IOError(msg.format(list(intersection)[0])) + + # Recursively check any contained superboxes. + for box in boxes: + if hasattr(box, 'box'): + _check_superbox_for_top_levels(box.box) + +def _check_top_level(boxes): + """Several boxes can only occur at the top level.""" + # Add the counts in the superboxes. + for box in boxes: + if hasattr(box, 'box'): + _check_superbox_for_top_levels(box.box) + + count = _collect_box_count(boxes) + # Which boxes occur more than once? + multiples = [box_id for box_id, bcount in count.items() if bcount > 1] + if 'dtbl' in multiples: + raise IOError('There can only be one dtbl box in a file.') + +def _check_for_singletons(boxes): + """Several boxes can only occur once.""" + count = _collect_box_count(boxes) + # Which boxes occur more than once? + multiples = [box_id for box_id, bcount in count.items() if bcount > 1] + if 'dtbl' in multiples: + raise IOError('There can only be one dtbl box in a file.') def _jpx_brand(boxes, brand): """ diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index dbbabb7..39151ba 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -67,6 +67,41 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(ET.tostring(jpx.box[-1].box[1].xml.getroot()), b'0') + def test_only_one_data_reference(self): + """Data reference boxes cannot be inside a superbox .""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 5]] + + flag = 0 + version = (0, 0, 0) + url = 'file:////usr/local/bin' + deurl = glymur.jp2box.DataEntryURLBox(flag, version, url) + dref = glymur.jp2box.DataReferenceBox([deurl]) + boxes.append(dref) + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jpx = jp2.wrap(tfile.name, boxes=boxes) + + def test_data_reference_not_at_top_level(self): + """Data reference boxes cannot be inside a superbox .""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 5]] + + flag = 0 + version = (0, 0, 0) + url = 'file:////usr/local/bin' + deurl = glymur.jp2box.DataEntryURLBox(flag, version, url) + dref = glymur.jp2box.DataReferenceBox([deurl]) + + # Put it inside the jp2 header box. + boxes[2].box.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jpx = jp2.wrap(tfile.name, boxes=boxes) + def test_jp2_to_jpx_sans_jp2_compatibility(self): """jp2 wrapped to jpx not including jp2 compatibility is wrong.""" jp2 = Jp2k(self.jp2file) @@ -133,46 +168,30 @@ class TestJPX(unittest.TestCase): def test_dtbl(self): """Verify that we can interpret Data Reference boxes.""" # Copy the existing JPX file, add a data reference box onto the end. + flag = 0 + version = (0, 0, 0) + url1 = 'file:////usr/local/bin' + url2 = 'http://glymur.readthedocs.org' + chr(0) * 3 with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: with open(self.jpxfile, 'rb') as ifile: tfile.write(ifile.read()) - # 8 + 2 + 20 + 36 - boxlen = 66 - write_buffer = struct.pack('>I4s', boxlen, b'dtbl') - tfile.write(write_buffer) - - # Just two boxes. - write_buffer = struct.pack('>H', 2) - tfile.write(write_buffer) - - # First data entry url box. - # This one will have a URL with 3 null chars at the end. - # They should be stripped off. - write_buffer = struct.pack('>I4s', 36, b'url ') - tfile.write(write_buffer) - url1 = 'file:////usr/local/bin' - write_buffer = struct.pack('>BBBB24s', 0, 0, 0, 0, - (url1 + chr(0) * 3).encode()) - tfile.write(write_buffer) - - # 2nd data entry url box. - write_buffer = struct.pack('>I4s', 20, b'url ') - tfile.write(write_buffer) - url2 = 'file:///' - write_buffer = struct.pack('>BBBB8s', 0, 0, 0, 0, url2.encode()) - tfile.write(write_buffer) + deurl1 = glymur.jp2box.DataEntryURLBox(flag, version, url1) + deurl2 = glymur.jp2box.DataEntryURLBox(flag, version, url2) + dref = glymur.jp2box.DataReferenceBox([deurl1, deurl2]) + dref.write(tfile) tfile.flush() - with self.assertWarns(UserWarning): + with warnings.catch_warnings(): + # This file has a rreq mask length that we do not recognize. + warnings.simplefilter("ignore") jpx = Jp2k(tfile.name) self.assertEqual(jpx.box[-1].box_id, 'dtbl') self.assertEqual(len(jpx.box[-1].DR), 2) self.assertEqual(jpx.box[-1].DR[0].url, url1) - self.assertEqual(jpx.box[-1].DR[1].url, url2) - + self.assertEqual(jpx.box[-1].DR[1].url, url2.rstrip('\0')) def test_ftbl(self): """Verify that we can interpret Fragment Table boxes.""" From 321a2544646495a159e45605f12769c6941c935d Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 8 Feb 2014 19:52:39 -0500 Subject: [PATCH 31/83] Completed XMP description. --- docs/source/how_do_i.rst | 72 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index b0b476c..c1a904c 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -262,7 +262,7 @@ following 'Google' But that would be painful. A better solution is to install the Python XMP -Toolkit:: +Toolkit (make sure it is version 2.0):: >>> from libxmp import XMPMeta >>> from libxmp.consts import XMP_NS_XMP as NS_XAP @@ -285,19 +285,77 @@ http://photojournal.jpl.nasa.gov/tiff/PIA17145.tif info JPEG 2000:: Next you can extract the XMP metadata. >>> from libxmp import XMPFiles - >>> xf = XMPFile() + >>> xf = XMPFiles() >>> xf.open_file('PIA17145.tif') >>> xmp = xf.get_xmp() + >>> print(xmp) + + + + + 1016 + 1016 + + + 8 + + + 1 + 1 + 1 + 1 + 2 + + + + + converted PNM file + + + + + + If you are familiar with TIFF, you can verify that there's no XMP tag in the TIFF file, but the Python XMP Toolkit takes advantage of the TIFF header structure to populate an XMP packet for you. If you were working with a JPEG file with Exif metadata, that information would be included in the XMP packet -as well. Now you can append the XMP packet in a UUIDBox:: +as well. Now you can append the XMP packet in a UUIDBox. In order to do this, +though, you have to know the UUID that signifies XMP data.:: >>> import uuid - >>> the_uuid = uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac') - >>> box = glymur.jp2box.UUIDBox(uuid, str(xmp).encode()) + >>> xmp_uuid = uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac') + >>> box = glymur.jp2box.UUIDBox(xmp_uuid, str(xmp).encode()) >>> jp2.append(box) - - + >>> print(jp2.box[-1]) + UUID Box (uuid) @ (592316, 1053) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + 1016 + 1016 + + + 8 + + + 1 + 1 + 1 + 1 + 2 + + + + + converted PNM file + + + + + From 4fafaa8661fe4352325a1577fbaf0dd4437e8e75 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 9 Feb 2014 12:51:28 -0500 Subject: [PATCH 32/83] Added set_printoptions, get_printoptions functions. Closes #158 --- CHANGES.txt | 9 +- glymur/__init__.py | 1 + glymur/jp2box.py | 183 ++++++++++++++++--- glymur/test/fixtures.py | 332 +++++++++++++++++++++++++++++++++++ glymur/test/test_printing.py | 162 +++++++---------- 5 files changed, 557 insertions(+), 130 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f70550e..f6c3728 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,10 +1,11 @@ -Feb 08, 2014 - Removed support for Python 2.6. Added write support for JP2 +Feb 09, 2014 - Removed support for Python 2.6. Added write support for JP2 UUID, DataEntryURL, Palette and Component Mapping boxes, JPX Association, NumberList and DataReference boxes. Added read support for JPX free, number list, data reference, fragment - table, and fragment list boxes. Palette box now a 2D numpy - array instead of a list of 1D arrays. JP2 super box constructors - now take optional box list argument. Fixed bug where JPX files + table, and fragment list boxes. Added get_printoptions, + set_printoptions functions. Palette box now a 2D numpy array + instead of a list of 1D arrays. JP2 super box constructors now + take optional box list argument. Fixed bug where JPX files with more than one codestream but advertising jp2 compatibility were not being read. diff --git a/glymur/__init__.py b/glymur/__init__.py index ba17727..5826f8c 100644 --- a/glymur/__init__.py +++ b/glymur/__init__.py @@ -8,6 +8,7 @@ __version__ = version.version from .jp2k import Jp2k from .jp2dump import jp2dump +from .jp2box import get_printoptions, set_printoptions from . import data diff --git a/glymur/jp2box.py b/glymur/jp2box.py index f8a0dc8..d0b6c07 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -14,9 +14,7 @@ References # pylint: disable=C0302,R0903,R0913 from collections import OrderedDict -import copy import datetime -import io import math import os import pprint @@ -24,7 +22,6 @@ import struct import sys import uuid import warnings -import xml import xml.etree.cElementTree as ET import numpy as np @@ -244,6 +241,8 @@ class ColourSpecificationBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg msg += '\n Method: {0}'.format(_METHOD_DISPLAY[self.method]) msg += '\n Precedence: {0}'.format(self.precedence) @@ -501,6 +500,9 @@ class ChannelDefinitionBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for j in range(len(self.association)): color_type_string = _COLOR_TYPE_MAP_DISPLAY[self.channel_type[j]] if self.association[j] == 0: @@ -591,6 +593,9 @@ class CodestreamHeaderBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for box in self.box: boxstr = str(box) @@ -655,6 +660,9 @@ class CompositingLayerHeaderBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for box in self.box: boxstr = str(box) @@ -727,6 +735,9 @@ class ComponentMappingBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for k in range(len(self.component_index)): if self.mapping_type[k] == 1: @@ -746,7 +757,7 @@ class ComponentMappingBox(Jp2kBox): fptr.write(write_buffer) for j in range(len(self.component_index)): - write_buffer = struct.pack('>HBB', + write_buffer = struct.pack('>HBB', self.component_index[j], self.mapping_type[j], self.palette_index[j]) @@ -812,6 +823,11 @@ class ContiguousCodestreamBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + if _printoptions['codestream'] == False: + return msg + msg += '\n Main header:' for segment in self.main_header.segment: segstr = str(segment) @@ -889,6 +905,9 @@ class DataReferenceBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for box in self.DR: msg += '\n ' + str(box) return msg @@ -976,7 +995,11 @@ class FileTypeBox(Jp2kBox): return msg def __str__(self): - lst = [Jp2kBox.__str__(self), + msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + + lst = [msg, ' Brand: {0}', ' Compatibility: {1}'] msg = '\n'.join(lst) @@ -1067,6 +1090,9 @@ class FragmentListBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for j in range(len(self.fragment_offset)): msg += "\n Offset {0}: {1}" msg += "\n Fragment Length {2}: {3}" @@ -1092,13 +1118,13 @@ class FragmentListBox(Jp2kBox): Returns ------- - FreeBox instance + FragmentListBox instance """ read_buffer = fptr.read(2) - nf, = struct.unpack('>H', read_buffer) + num_fragments, = struct.unpack('>H', read_buffer) - read_buffer = fptr.read(nf * 14) - lst = struct.unpack('>' + 'QIH' * nf, read_buffer) + read_buffer = fptr.read(num_fragments * 14) + lst = struct.unpack('>' + 'QIH' * num_fragments, read_buffer) frag_offset = lst[0::3] frag_len = lst[1::3] data_reference = lst[2::3] @@ -1131,6 +1157,9 @@ class FragmentTableBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for box in self.box: boxstr = str(box) @@ -1191,6 +1220,9 @@ class FreeBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + return msg @staticmethod @@ -1280,6 +1312,10 @@ class ImageHeaderBox(Jp2kBox): return msg def __str__(self): + msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg = "{0}" msg += '\n Size: [{1} {2} {3}]' msg += '\n Bitdepth: {4}' @@ -1380,6 +1416,9 @@ class AssociationBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for box in self.box: boxstr = str(box) @@ -1513,6 +1552,9 @@ class JPEG2000SignatureBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n Signature: {0:02x}{1:02x}{2:02x}{3:02x}' msg = msg.format(self.signature[0], self.signature[1], self.signature[2], self.signature[3]) @@ -1584,16 +1626,15 @@ class PaletteBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n Size: ({0} x {1})'.format(*self.palette.shape) return msg def write(self, fptr): """Write a Palette box to file. """ - # Box length is usual header (8) - # + num entries NE (2) + num columns NC (1) - # + (bps/8, /signed) for each column (3) + bps * NC - # + bytes_per_row = sum(self.bits_per_component) / 8 bytes_per_palette = bytes_per_row * self.palette.shape[0] box_length = 8 + 3 + self.palette.shape[1] + bytes_per_palette @@ -1608,7 +1649,7 @@ class PaletteBox(Jp2kBox): fptr.write(write_buffer) bps_signed = [x - 1 for x in self.bits_per_component] - for j, item in enumerate(bps_signed): + for j, _ in enumerate(bps_signed): if self.signed[j]: bps_signed[j] |= 0x80 write_buffer = struct.pack('>' + 'B' * self.palette.shape[1], @@ -1616,13 +1657,10 @@ class PaletteBox(Jp2kBox): fptr.write(write_buffer) if self.bits_per_component[0] <= 8: - dtype = np.uint8 code = 'B' elif self.bits_per_component[0] <= 16: - dtype = np.uint16 code = 'H' elif self.bits_per_component[0] <= 32: - dtype = np.uint32 code = 'I' fmt = '>' + code * self.palette.shape[1] @@ -1820,6 +1858,9 @@ class ReaderRequirementsBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n Standard Features:' for j in range(len(self.standard_flag)): @@ -2044,6 +2085,9 @@ class CaptureResolutionBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n VCR: {0}'.format(self.vertical_resolution) msg += '\n HCR: {0}'.format(self.horizontal_resolution) return msg @@ -2106,6 +2150,9 @@ class DisplayResolutionBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n VDR: {0}'.format(self.vertical_resolution) msg += '\n HDR: {0}'.format(self.horizontal_resolution) return msg @@ -2162,6 +2209,9 @@ class LabelBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n Label: {0}'.format(self.label) return msg @@ -2218,6 +2268,9 @@ class NumberListBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for j, association in enumerate(self.associations): msg += '\n Association[{0}]: '.format(j) if association == 0: @@ -2315,6 +2368,11 @@ class XMLBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + if _printoptions['xml'] == False: + return msg + xml = self.xml if self.xml is not None: msg += _pretty_print_xml(self.xml) @@ -2416,9 +2474,12 @@ class UUIDListBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + for j, uuid_item in enumerate(self.ulst): msg += '\n UUID[{0}]: {1}'.format(j, uuid_item) - return(msg) + return msg @staticmethod def parse(fptr, offset, length): @@ -2446,7 +2507,7 @@ class UUIDListBox(Jp2kBox): ulst.append(uuid.UUID(bytes=read_buffer)) box = UUIDListBox(ulst, length=length, offset=offset) - return(box) + return box class UUIDInfoBox(Jp2kBox): @@ -2477,7 +2538,6 @@ class UUIDInfoBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) - for box in self.box: box_str = str(box) @@ -2485,7 +2545,7 @@ class UUIDInfoBox(Jp2kBox): lst = [('\n ' + x) for x in box_str.split('\n')] msg += ''.join(lst) - return(msg) + return msg @staticmethod def parse(fptr, offset, length): @@ -2561,6 +2621,9 @@ class DataEntryURLBox(Jp2kBox): def __str__(self): msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + msg += '\n ' lines = ['Version: {0}', @@ -2671,15 +2734,30 @@ class UUIDBox(Jp2kBox): return msg.format(repr(self.uuid), len(self.raw_data)) def __str__(self): - msg = '{0}\n UUID: {1}'.format(Jp2kBox.__str__(self), self.uuid) + msg = Jp2kBox.__str__(self) + if _printoptions['short'] == True: + return msg + + msg = '{0}\n UUID: {1}'.format(msg, self.uuid) + if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): + msg += ' (XMP)' + elif self.uuid.bytes == b'JpgTiffExif->JP2': + msg += ' (EXIF)' + else: + msg += ' (unknown)' + + if (((_printoptions['xml'] == False) and + (self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac')))): + # If it's an XMP UUID, don't print the XML contents. + return msg if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - line = ' (XMP)\n UUID Data: {0}' + line = '\n UUID Data: {0}' msg += line.format(_pretty_print_xml(self.data)) elif self.uuid.bytes == b'JpgTiffExif->JP2': - msg += ' (EXIF)\n UUID Data: {0}'.format(str(self.data)) + msg += '\n UUID Data: {0}'.format(str(self.data)) else: - line = ' (unknown)\n UUID Data: {0} bytes' + line = '\n UUID Data: {0} bytes' msg += line.format(len(self.raw_data)) return msg @@ -2748,3 +2826,58 @@ _BOX_WITH_ID = { 'url ': DataEntryURLBox, 'uuid': UUIDBox, 'xml ': XMLBox} + +_printoptions = {'short': False, 'xml': True, 'codestream': True} + +def set_printoptions(short=False, xml=True, codestream=True): + """Set printing options. + + These options determine the way JPEG 2000 boxes are displayed. + + Parameters + ---------- + short : bool, optional + When True, only the box ID, offset, and length are displayed. Useful + for displaying only the basic structure or skeleton of a JPEG 2000 file. + xml : bool, optional + When False, printing of the XML contents of any XML boxes or UUID XMP + boxes is suppressed. + codestream : bool, optional + When False, printing of the codestream contents is suppressed. + + See also + -------- + get_printoptions + + Examples + -------- + To put back the default options, you can use: + + >>> import glymur + >>> glymur.set_printoptions(short=False, xml=True, codestream=True) + """ + _printoptions['short'] = short + _printoptions['xml'] = xml + _printoptions['codestream'] = codestream + +def get_printoptions(): + """Return the current print options. + + Returns + ------- + print_opts : dict + Dictionary of current print options with keys + + - short : bool + - xml : bool + - codestream : bool + + For a full description of these options, see `set_printoptions`. + + See also + -------- + set_printoptions + """ + return _printoptions + + diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index ff33237..182a04f 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -350,3 +350,335 @@ text_gbr_34 = """Colour Specification Box (colr) @ (179, 1339) 'Rendering Intent': 'perceptual', 'Illuminant': array([ 0.96420288, 1. , 0.8249054 ]), 'Creator': 'appl'}""" + + +# Metadata dump of nemo. +nemo_dump_full = r'''JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a +File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] +JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB +UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + + +Contiguous Codestream Box (jp2c) @ (3223, 1132296) + Main header: + SOC marker segment @ (3231, 0) + SIZ marker segment @ (3233, 47) + Profile: 2 + Reference Grid Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Grid Offset: (0 x 0) + Reference Tile Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Tile Offset: (0 x 0) + Bitdepth: (8, 8, 8) + Signed: (False, False, False) + Vertical, Horizontal Subsampling: ((1, 1), (1, 1), (1, 1)) + COD marker segment @ (3282, 12) + Coding style: + Entropy coder, without partitions + SOP marker segments: False + EPH marker segments: False + Coding style parameters: + Progression order: LRCP + Number of layers: 2 + Multiple component transformation usage: reversible + Number of resolutions: 2 + Code block height, width: (64 x 64) + Wavelet transform: 5-3 reversible + Precinct size: default, 2^15 x 2^15 + Code block context: + Selective arithmetic coding bypass: False + Reset context probabilities on coding pass boundaries: False + Termination on each coding pass: False + Vertically stripe causal context: False + Predictable termination: False + Segmentation symbols: False + QCD marker segment @ (3296, 7) + Quantization style: no quantization, 2 guard bits + Step size: [(0, 8), (0, 9), (0, 9), (0, 10)] + CME marker segment @ (3305, 37) + "Created by OpenJPEG version 2.0.0"''' + +nemo_dump_short = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) +File Type Box (ftyp) @ (12, 20) +JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Colour Specification Box (colr) @ (62, 15) +UUID Box (uuid) @ (77, 3146) +Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" + +nemo_dump_no_xml = '''JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a +File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] +JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB +UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) +Contiguous Codestream Box (jp2c) @ (3223, 1132296) + Main header: + SOC marker segment @ (3231, 0) + SIZ marker segment @ (3233, 47) + Profile: 2 + Reference Grid Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Grid Offset: (0 x 0) + Reference Tile Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Tile Offset: (0 x 0) + Bitdepth: (8, 8, 8) + Signed: (False, False, False) + Vertical, Horizontal Subsampling: ((1, 1), (1, 1), (1, 1)) + COD marker segment @ (3282, 12) + Coding style: + Entropy coder, without partitions + SOP marker segments: False + EPH marker segments: False + Coding style parameters: + Progression order: LRCP + Number of layers: 2 + Multiple component transformation usage: reversible + Number of resolutions: 2 + Code block height, width: (64 x 64) + Wavelet transform: 5-3 reversible + Precinct size: default, 2^15 x 2^15 + Code block context: + Selective arithmetic coding bypass: False + Reset context probabilities on coding pass boundaries: False + Termination on each coding pass: False + Vertically stripe causal context: False + Predictable termination: False + Segmentation symbols: False + QCD marker segment @ (3296, 7) + Quantization style: no quantization, 2 guard bits + Step size: [(0, 8), (0, 9), (0, 9), (0, 10)] + CME marker segment @ (3305, 37) + "Created by OpenJPEG version 2.0.0"''' + +nemo_dump_no_codestream = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a +File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] +JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB +UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + + +Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" + +nemo_dump_no_codestream_no_xml = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a +File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] +JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB +UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) +Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 3e9d3ab..fa9ab0c 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -11,7 +11,6 @@ # pylint: disable=R0904 import os -import re import struct import sys import tempfile @@ -31,104 +30,76 @@ else: import glymur from glymur import Jp2k +from . import fixtures from .fixtures import OPJ_DATA_ROOT, opj_data_file, nemo_xmp_box from .fixtures import text_gbr_27, text_gbr_33, text_gbr_34 @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") -@unittest.skipIf(re.match(r"""1\.[01234]""", glymur.version.openjpeg_version), - "Need at least 1.5 in order to write jp2 files.") -class TestPrintingNeedsLib(unittest.TestCase): - """These tests require the library, mostly in order to just setup the test. - """ - - @classmethod - def setUpClass(cls): - # Setup a plain JP2 file without the two UUID boxes. - jp2file = glymur.data.nemo() - with tempfile.NamedTemporaryFile(suffix='.jp2', delete=False) as tfile: - cls._plain_nemo_file = tfile.name - ijfile = Jp2k(jp2file) - data = ijfile.read(rlevel=1) - ojfile = Jp2k(cls._plain_nemo_file, 'wb') - ojfile.write(data) - - @classmethod - def tearDownClass(cls): - os.unlink(cls._plain_nemo_file) - +class TestPrinting(unittest.TestCase): + """Tests for verifying how printing works.""" def setUp(self): self.jp2file = glymur.data.nemo() self.j2kfile = glymur.data.goodstuff() - # Save the output of dumping nemo.jp2 for more than one test. - lines = ['JPEG 2000 Signature Box (jP ) @ (0, 12)', - ' Signature: 0d0a870a', - 'File Type Box (ftyp) @ (12, 20)', - ' Brand: jp2 ', - " Compatibility: ['jp2 ']", - 'JP2 Header Box (jp2h) @ (32, 45)', - ' Image Header Box (ihdr) @ (40, 22)', - ' Size: [728 1296 3]', - ' Bitdepth: 8', - ' Signed: False', - ' Compression: wavelet', - ' Colorspace Unknown: False', - ' Colour Specification Box (colr) @ (62, 15)', - ' Method: enumerated colorspace', - ' Precedence: 0', - ' Colorspace: sRGB', - 'Contiguous Codestream Box (jp2c) @ (77, 1632355)', - ' Main header:', - ' SOC marker segment @ (85, 0)', - ' SIZ marker segment @ (87, 47)', - ' Profile: 2', - ' Reference Grid Height, Width: (728 x 1296)', - ' Vertical, Horizontal Reference Grid Offset: ' - + '(0 x 0)', - ' Reference Tile Height, Width: (728 x 1296)', - ' Vertical, Horizontal Reference Tile Offset: ' - + '(0 x 0)', - ' Bitdepth: (8, 8, 8)', - ' Signed: (False, False, False)', - ' Vertical, Horizontal Subsampling: ' - + '((1, 1), (1, 1), (1, 1))', - ' COD marker segment @ (136, 12)', - ' Coding style:', - ' Entropy coder, without partitions', - ' SOP marker segments: False', - ' EPH marker segments: False', - ' Coding style parameters:', - ' Progression order: LRCP', - ' Number of layers: 1', - ' Multiple component transformation usage: ' - + 'reversible', - ' Number of resolutions: 6', - ' Code block height, width: (64 x 64)', - ' Wavelet transform: 5-3 reversible', - ' Precinct size: default, 2^15 x 2^15', - ' Code block context:', - ' Selective arithmetic coding bypass: ' - + 'False', - ' Reset context probabilities on ' - + 'coding pass boundaries: False', - ' Termination on each coding pass: False', - ' Vertically stripe causal context: ' - + 'False', - ' Predictable termination: False', - ' Segmentation symbols: False', - ' QCD marker segment @ (150, 19)', - ' Quantization style: no quantization, ' - + '2 guard bits', - ' Step size: [(0, 8), (0, 9), (0, 9), ' - + '(0, 10), (0, 9), (0, 9), (0, 10), (0, 9), (0, 9), ' - + '(0, 10), (0, 9), (0, 9), (0, 10), (0, 9), (0, 9), ' - + '(0, 10)]'] - self.expected_plain = '\n'.join(lines) + # Reset printoptions for every test. + glymur.set_printoptions(short=False, xml=True, codestream=True) def tearDown(self): pass + def test_printopt_no_codestr_or_xml(self): + """Verify printed output when codestream=False and xml=False""" + glymur.set_printoptions(codestream=False, xml=False) + with patch('sys.stdout', new=StringIO()) as fake_out: + glymur.jp2dump(self.jp2file) + actual = fake_out.getvalue().strip() + + # Get rid of the filename line, as it is not set in stone. + lst = actual.split('\n') + lst = lst[1:] + actual = '\n'.join(lst) + self.assertEqual(actual, fixtures.nemo_dump_no_codestream_no_xml) + + def test_printoptions_no_codestream(self): + """Verify printed output when codestream=False""" + glymur.set_printoptions(codestream=False) + with patch('sys.stdout', new=StringIO()) as fake_out: + glymur.jp2dump(self.jp2file) + actual = fake_out.getvalue().strip() + + # Get rid of the filename line, as it is not set in stone. + lst = actual.split('\n') + lst = lst[1:] + actual = '\n'.join(lst) + self.assertEqual(actual, fixtures.nemo_dump_no_codestream) + + def test_printoptions_no_xml(self): + """Verify printed output when xml=False""" + glymur.set_printoptions(xml=False) + with patch('sys.stdout', new=StringIO()) as fake_out: + glymur.jp2dump(self.jp2file) + actual = fake_out.getvalue().strip() + + # Get rid of the filename line, as it is not set in stone. + lst = actual.split('\n') + lst = lst[1:] + actual = '\n'.join(lst) + self.assertEqual(actual, fixtures.nemo_dump_no_xml) + + def test_printoptions_short(self): + """Verify printed output when short=True""" + glymur.set_printoptions(short=True) + with patch('sys.stdout', new=StringIO()) as fake_out: + glymur.jp2dump(self.jp2file) + actual = fake_out.getvalue().strip() + + # Get rid of the filename line, as it is not set in stone. + lst = actual.split('\n') + lst = lst[1:] + actual = '\n'.join(lst) + self.assertEqual(actual, fixtures.nemo_dump_short) + def test_asoc_label_box(self): """verify printing of asoc, label boxes""" # Construct a fake file with an asoc and a label box, as @@ -185,18 +156,18 @@ class TestPrintingNeedsLib(unittest.TestCase): def test_jp2dump(self): """basic jp2dump test""" with patch('sys.stdout', new=StringIO()) as fake_out: - glymur.jp2dump(self._plain_nemo_file) + glymur.jp2dump(self.jp2file) actual = fake_out.getvalue().strip() # Get rid of the filename line, as it is not set in stone. lst = actual.split('\n') lst = lst[1:] actual = '\n'.join(lst) - self.assertEqual(actual, self.expected_plain) + self.assertEqual(actual, fixtures.nemo_dump_full) def test_entire_file(self): """verify output from printing entire file""" - j = glymur.Jp2k(self._plain_nemo_file) + j = glymur.Jp2k(self.jp2file) with patch('sys.stdout', new=StringIO()) as fake_out: print(j) actual = fake_out.getvalue().strip() @@ -206,18 +177,7 @@ class TestPrintingNeedsLib(unittest.TestCase): lst = lst[1:] actual = '\n'.join(lst) - self.assertEqual(actual, self.expected_plain) - - -class TestPrinting(unittest.TestCase): - """Test suite for printing where the libraries are not needed""" - - def setUp(self): - # Save sys.stdout. - self.jp2file = glymur.data.nemo() - - def tearDown(self): - pass + self.assertEqual(actual, fixtures.nemo_dump_full) def test_coc_segment(self): """verify printing of COC segment""" From 47d8e8470bd251bbc149ee21f79e5d84fba0dfee Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 06:58:11 -0500 Subject: [PATCH 33/83] Print options remaining persistent across invocations. #162 --- glymur/jp2box.py | 9 +++++---- glymur/test/test_printing.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index d0b6c07..12613fa 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2829,7 +2829,7 @@ _BOX_WITH_ID = { _printoptions = {'short': False, 'xml': True, 'codestream': True} -def set_printoptions(short=False, xml=True, codestream=True): +def set_printoptions(**kwargs): """Set printing options. These options determine the way JPEG 2000 boxes are displayed. @@ -2856,9 +2856,10 @@ def set_printoptions(short=False, xml=True, codestream=True): >>> import glymur >>> glymur.set_printoptions(short=False, xml=True, codestream=True) """ - _printoptions['short'] = short - _printoptions['xml'] = xml - _printoptions['codestream'] = codestream + for key, value in kwargs.items(): + if key not in ['short', 'xml', 'codestream']: + raise TypeError('"{0}" not a valid keyword parameter.'.format(key)) + _printoptions[key] = value def get_printoptions(): """Return the current print options. diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index fa9ab0c..56fa4f0 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -48,6 +48,26 @@ class TestPrinting(unittest.TestCase): def tearDown(self): pass + def test_printoptions_bad_argument(self): + """Verify error when bad parameter to set_printoptions""" + with self.assertRaises(TypeError): + glymur.set_printoptions(hi='low') + + def test_printopt_no_codestr_then_no_xml(self): + """Verify printed output when codestream=False and xml=False, #162""" + # The print options should be persistent across invocations. + glymur.set_printoptions(codestream=False) + glymur.set_printoptions(xml=False) + with patch('sys.stdout', new=StringIO()) as fake_out: + glymur.jp2dump(self.jp2file) + actual = fake_out.getvalue().strip() + + # Get rid of the filename line, as it is not set in stone. + lst = actual.split('\n') + lst = lst[1:] + actual = '\n'.join(lst) + self.assertEqual(actual, fixtures.nemo_dump_no_codestream_no_xml) + def test_printopt_no_codestr_or_xml(self): """Verify printed output when codestream=False and xml=False""" glymur.set_printoptions(codestream=False, xml=False) From 29dbf594a5fc7354409bbef43635ba98d14b619e Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 09:38:15 -0500 Subject: [PATCH 34/83] Now printing fuam, dcm, standard masks. #163 Improves visual decoding of the reader requirements box. --- CHANGES.txt | 12 ++++++------ glymur/jp2box.py | 8 ++++++++ glymur/test/fixtures.py | 14 ++++++++++++++ glymur/test/test_printing.py | 15 +-------------- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index f6c3728..1febdfb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -2,12 +2,12 @@ Feb 09, 2014 - Removed support for Python 2.6. Added write support for JP2 UUID, DataEntryURL, Palette and Component Mapping boxes, JPX Association, NumberList and DataReference boxes. Added read support for JPX free, number list, data reference, fragment - table, and fragment list boxes. Added get_printoptions, - set_printoptions functions. Palette box now a 2D numpy array - instead of a list of 1D arrays. JP2 super box constructors now - take optional box list argument. Fixed bug where JPX files - with more than one codestream but advertising jp2 compatibility - were not being read. + table, and fragment list boxes. Improved JPX Reader Requirements box + support. Added get_printoptions, set_printoptions functions. + Palette box now a 2D numpy array instead of a list of 1D arrays. + JP2 super box constructors now take optional box list argument. + Fixed bug where JPX files with more than one codestream but + advertising jp2 compatibility were not being read. Jan 28, 2014 - v0.5.10 Fixed bad warning when reader requirements box mask length is unsupported. diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 12613fa..53bad9f 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1861,6 +1861,14 @@ class ReaderRequirementsBox(Jp2kBox): if _printoptions['short'] == True: return msg + msg += '\n Fully Understands Aspect Mask: {0}'.format(self.fuam) + msg += '\n Display Completely Mask: {0}'.format(self.dcm) + + msg += '\n Standard Features and Masks:' + for j in range(len(self.standard_flag)): + sfl = self.standard_flag[j] + mask = self.standard_mask[j] + msg += '\n Feature {0:03d}: {1}'.format(sfl, mask) msg += '\n Standard Features:' for j in range(len(self.standard_flag)): diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 182a04f..0d5069b 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -682,3 +682,17 @@ JP2 Header Box (jp2h) @ (32, 45) UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" + +# Output of reader requirement printing for file7.jp2 +file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) + Fully Understands Aspect Mask: 160 + Display Completely Mask: 192 + Standard Features and Masks: + Feature 005: 128 + Feature 060: 96 + Feature 043: 64 + Standard Features: + Feature 005: Unrestricted JPEG 2000 Part 1 codestream, ITU-T Rec. T.800 | ISO/IEC 15444-1 + Feature 060: e-sRGB enumerated colorspace + Feature 043: (Deprecated) compositing layer uses restricted ICC profile + Vendor Features:""" diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 56fa4f0..26156dc 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -738,20 +738,7 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(j.box[2]) actual = fake_out.getvalue().strip() - lines = ['Reader Requirements Box (rreq) @ (44, 24)', - ' Standard Features:', - ' Feature 005: ' - + 'Unrestricted JPEG 2000 Part 1 codestream, ' - + 'ITU-T Rec. T.800 | ISO/IEC 15444-1', - ' Feature 060: e-sRGB enumerated colorspace', - - ' Feature 043: ' - + '(Deprecated) ' - + 'compositing layer uses restricted ICC profile', - - ' Vendor Features:'] - expected = '\n'.join(lines) - self.assertEqual(actual, expected) + self.assertEqual(actual, fixtures.file7_rreq) @unittest.skipIf(OPJ_DATA_ROOT is None, "OPJ_DATA_ROOT environment variable not set") From 23e5f4db3fd0954a24e8ed19e5ce8a709f63f0bc Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 11:38:54 -0500 Subject: [PATCH 35/83] Adding hooks into set_printoptions. --- bin/jp2dump | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/bin/jp2dump b/bin/jp2dump index cb05c44..f419787 100755 --- a/bin/jp2dump +++ b/bin/jp2dump @@ -5,11 +5,32 @@ import sys import glymur description='Print JPEG2000 metadata.' +epilog=r"""The default action is to print just the codestream headers. Supplying +--nocodestream will prevent any codestream information from printing, while +--fullcodestream will print the entire codestream (and just the codestream). + +JP2 file: default is to suppress codestream and XML. +JPX file: default is to suppress codestream and XML. +JPC file: default is to print codestream header. +""" parser = argparse.ArgumentParser(description=description) -parser.add_argument('-c', '--codestream', help='dump codestream', - action='store_true') +parser.add_argument('-x', '--noxml', help='suppress xml', + action='store_true') +parser.add_argument('--short', help='only box id, offset, and length', + action='store_true') +parser.add_argument('-c', '--nocodestream', help='suppress codestream', + action='store_false') +parser.add_argument('-f', '--fullcodestream', help='full codestream', + action='store_true') parser.add_argument('filename') args = parser.parse_args() +print(args) +if args.noxml: + glymur.set_printoptions(xml=False) +if args.short: + glymur.set_printoptions(short=True) +if args.nocodestream: + glymur.set_printoptions(codestream=False) filename = args.filename -glymur.jp2dump(args.filename, codestream=args.codestream) +glymur.jp2dump(args.filename, codestream=args.fullcodestream) From 116ef6e664ab35652fd9dc5da8fd5f24e3bccb66 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 11:48:16 -0500 Subject: [PATCH 36/83] _uuid_io no longer a sub package but rather a module of glymur proper. #166 --- glymur/{_uuid_io/Exif.py => _uuid_io.py} | 2 +- glymur/_uuid_io/__init__.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) rename glymur/{_uuid_io/Exif.py => _uuid_io.py} (99%) delete mode 100644 glymur/_uuid_io/__init__.py diff --git a/glymur/_uuid_io/Exif.py b/glymur/_uuid_io.py similarity index 99% rename from glymur/_uuid_io/Exif.py rename to glymur/_uuid_io.py index b07cd42..d4971a1 100644 --- a/glymur/_uuid_io/Exif.py +++ b/glymur/_uuid_io.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Handlers for Exif UUIDs. Be nice if we would find a standard for this. +Part of glymur. """ import pprint import re diff --git a/glymur/_uuid_io/__init__.py b/glymur/_uuid_io/__init__.py deleted file mode 100644 index 778f912..0000000 --- a/glymur/_uuid_io/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -Sub package for handling various types of UUIDs. -""" -from .Exif import tiff_header, xml From f2f05652e4a630665e1fbcd7398432371e4a5b00 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 14:52:53 -0500 Subject: [PATCH 37/83] Added printoption support to jp2dump. #164 --- bin/jp2dump | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/bin/jp2dump b/bin/jp2dump index f419787..08907e5 100755 --- a/bin/jp2dump +++ b/bin/jp2dump @@ -5,32 +5,30 @@ import sys import glymur description='Print JPEG2000 metadata.' -epilog=r"""The default action is to print just the codestream headers. Supplying ---nocodestream will prevent any codestream information from printing, while ---fullcodestream will print the entire codestream (and just the codestream). - -JP2 file: default is to suppress codestream and XML. -JPX file: default is to suppress codestream and XML. -JPC file: default is to print codestream header. -""" parser = argparse.ArgumentParser(description=description) -parser.add_argument('-x', '--noxml', help='suppress xml', +parser.add_argument('-x', '--noxml', help='Suppress XML.', action='store_true') -parser.add_argument('--short', help='only box id, offset, and length', - action='store_true') -parser.add_argument('-c', '--nocodestream', help='suppress codestream', - action='store_false') -parser.add_argument('-f', '--fullcodestream', help='full codestream', +parser.add_argument('-s', '--short', help='Only print box id, offset, and length.', action='store_true') +chelp='Level of codestream information. 0 suppressed all details, 1 prints headers, 2 prints the full codestream' +parser.add_argument('-c', '--codestream', + help=chelp, + nargs=1, + type=int, + default=0) parser.add_argument('filename') args = parser.parse_args() -print(args) if args.noxml: glymur.set_printoptions(xml=False) if args.short: glymur.set_printoptions(short=True) -if args.nocodestream: +if args.codestream[0] == 0: glymur.set_printoptions(codestream=False) + print_full_codestream = False +elif args.codestream[0] == 1: + print_full_codestream = False +else: + print_full_codestream = True filename = args.filename -glymur.jp2dump(args.filename, codestream=args.fullcodestream) +glymur.jp2dump(args.filename, codestream=print_full_codestream) From 3f56a08d2ced7a18b2d539daccd0222e4a1fa308 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 15:13:42 -0500 Subject: [PATCH 38/83] Added usage of set_printoptions method. #161 --- docs/source/how_do_i.rst | 197 +++++++++++++++++++++++++++++++++++---- 1 file changed, 178 insertions(+), 19 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index 01dd24c..dde5b60 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -11,8 +11,8 @@ resolution level. :: >>> import glymur >>> file = glymur.data.nemo() - >>> j = glymur.Jp2k(file) - >>> thumbnail = j.read(rlevel=-1) + >>> jp2 = glymur.Jp2k(file) + >>> thumbnail = jp2.read(rlevel=-1) ... display metadata? ===================== @@ -23,16 +23,183 @@ available. :: From within Python, it is as simple as printing the Jp2k object, i.e. :: - >>> from glymur import Jp2k + >>> import glymur >>> file = glymur.data.nemo() - >>> j = Jp2k(file) - >>> print(j) + >>> jp2 = glymur.Jp2k(file) + >>> print(jp2) + File: nemo.jp2 + JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a + File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] + JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB + UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + + + Contiguous Codestream Box (jp2c) @ (3223, 1132296) + Main header: + SOC marker segment @ (3231, 0) + SIZ marker segment @ (3233, 47) + Profile: 2 + Reference Grid Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Grid Offset: (0 x 0) + Reference Tile Height, Width: (1456 x 2592) + Vertical, Horizontal Reference Tile Offset: (0 x 0) + Bitdepth: (8, 8, 8) + Signed: (False, False, False) + Vertical, Horizontal Subsampling: ((1, 1), (1, 1), (1, 1)) + COD marker segment @ (3282, 12) + Coding style: + Entropy coder, without partitions + SOP marker segments: False + EPH marker segments: False + Coding style parameters: + Progression order: LRCP + Number of layers: 2 + Multiple component transformation usage: reversible + Number of resolutions: 2 + Code block height, width: (64 x 64) + Wavelet transform: 5-3 reversible + Precinct size: default, 2^15 x 2^15 + Code block context: + Selective arithmetic coding bypass: False + Reset context probabilities on coding pass boundaries: False + Termination on each coding pass: False + Vertically stripe causal context: False + Predictable termination: False + Segmentation symbols: False + QCD marker segment @ (3296, 7) + Quantization style: no quantization, 2 guard bits + Step size: [(0, 8), (0, 9), (0, 9), (0, 10)] + CME marker segment @ (3305, 37) + "Created by OpenJPEG version 2.0.0" + +That's fairly overwhelming, and perhaps lost in the flood of information +is the fact that the codestream metadata is limited to just what's in the +main codestream header. You can suppress the codestream and XML details by +making use of the :py:meth:`set_printoptions` function:: -This prints the metadata found in the JP2 boxes, but in the case of the -codestream box, only the main header is printed. It is possible to print -**only** the codestream information as well, i.e. :: + >>> glymur.set_printoptions(codestream=False, xml=False) + >>> print(jp2) + File: nemo.jp2 + JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a + File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] + JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [1456 2592 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB + UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + Contiguous Codestream Box (jp2c) @ (3223, 1132296) - >>> print(j.get_codestream()) +It is possible to print all the gory codestream details as well, i.e. :: + + >>> print(j.get_codestream()) # details not shown ... add XML metadata? ===================== @@ -55,7 +222,7 @@ Consider the following XML file `data.xml` : :: -The **append** method can add an XML box as shown below:: +The :py:meth:`append` method can add an XML box as shown below:: >>> import shutil >>> import glymur @@ -74,6 +241,7 @@ codestream provided by `goodstuff.j2k` (a file consisting of a raw codestream), you can use the :py:meth:`wrap` method with no box argument: :: >>> import glymur + >>> glymur.set_printoptions(codestream=False) >>> jfile = glymur.data.goodstuff() >>> j2k = glymur.Jp2k(jfile) >>> jp2 = j2k.wrap("newfile.jp2") @@ -96,10 +264,6 @@ you can use the :py:meth:`wrap` method with no box argument: :: Precedence: 0 Colorspace: sRGB Contiguous Codestream Box (jp2c) @ (77, 115228) - Main header: - . - . (truncated) - . The raw codestream was wrapped in a JP2 jacket with four boxes in the outer layer (the signature, file type, JP2 header, and contiguous codestream), with @@ -146,12 +310,7 @@ the following will work. :: Light Ale - Contiguous Codestream Box (jp2c) @ (153, 115236) - Main header: - . - . (truncated) - . As to the question of which method you should use, :py:meth:`append` or :py:meth:`wrap`, to add metadata, you should keep in mind that :py:meth:`wrap` From 1484fd974f0dc31cd9b7fe9abe5318be8e95748e Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 15:43:18 -0500 Subject: [PATCH 39/83] Removed trailing newline from XML string output. #160 --- glymur/jp2box.py | 4 +++- glymur/test/fixtures.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 53bad9f..07ddb6e 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2761,7 +2761,9 @@ class UUIDBox(Jp2kBox): if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): line = '\n UUID Data: {0}' - msg += line.format(_pretty_print_xml(self.data)) + xmlstring = _pretty_print_xml(self.data) + xmlstring = xmlstring.rstrip() + msg += line.format(xmlstring) elif self.uuid.bytes == b'JpgTiffExif->JP2': msg += '\n UUID Data: {0}'.format(str(self.data)) else: diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 0d5069b..5f7d77a 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -455,7 +455,6 @@ UUID Box (uuid) @ (77, 3146) - Contiguous Codestream Box (jp2c) @ (3223, 1132296) Main header: SOC marker segment @ (3231, 0) @@ -660,7 +659,6 @@ UUID Box (uuid) @ (77, 3146) - Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" nemo_dump_no_codestream_no_xml = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) From 87cbf87ed76f7d2000cc6c5381964dbd9f4fb765 Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 10 Feb 2014 16:47:09 -0500 Subject: [PATCH 40/83] Checking for proper location of jp2h children. #157 Certain boxes can only reside in jp2h or jpch (which is not currently supported for writing purposed). --- glymur/jp2k.py | 16 ++++++++++++++++ glymur/test/test_jp2box.py | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 3ecc96f..55e3766 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1198,6 +1198,22 @@ def _validate_jp2_box_sequence(boxes): _jpx_compatibility(boxes, boxes[1].compatibility_list) _check_for_singletons(boxes) _check_top_level(boxes) + _check_jp2h_child_boxes(boxes, 'top-level') + +JP2H_CHILDREN = set(['bpcc', 'cmap', 'ihdr', 'pclr']) +def _check_jp2h_child_boxes(boxes, parent_box_name): + """Certain boxes can only reside in the JP2 header.""" + box_ids = set([box.box_id for box in boxes]) + intersection = box_ids.intersection(JP2H_CHILDREN) + if len(intersection) > 0 and parent_box_name != 'jp2h': + msg = "A '{0}' box can only be nested in a JP2 header box." + raise IOError(msg.format(list(intersection)[0])) + + # Recursively check any contained superboxes. + for box in boxes: + if hasattr(box, 'box'): + _check_jp2h_child_boxes(box.box, box.box_id) + def _collect_box_count(boxes): """Count the occurences of each box type.""" diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 23c8e93..ac1b2a3 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -679,6 +679,33 @@ class TestWrap(unittest.TestCase): with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) + def test_pclr_not_in_jp2h(self): + """A palette box must reside in a JP2 header box.""" + palette = np.array([[255, 0, 255], [0, 255, 0]], dtype=np.int32) + bps = (8, 8, 8) + signed = (True, False, True) + pclr = glymur.jp2box.PaletteBox(palette=palette, bits_per_component=bps, + signed=(True, False, True)) + + j2k = Jp2k(self.j2kfile) + codestream = j2k.get_codestream() + height = codestream.segment[1].ysiz + width = codestream.segment[1].xsiz + num_components = len(codestream.segment[1].xrsiz) + + jp2b = JPEG2000SignatureBox() + ftyp = FileTypeBox() + jp2h = JP2HeaderBox() + jp2c = ContiguousCodestreamBox() + colr = ColourSpecificationBox(colorspace=glymur.core.SRGB) + ihdr = ImageHeaderBox(height=height, width=width, + num_components=num_components) + jp2h.box = [ihdr, colr] + boxes = [jp2b, ftyp, jp2h, jp2c, pclr] + with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: + with self.assertRaises(IOError): + j2k.wrap(tfile.name, boxes=boxes) + def test_jp2h_not_preceeding_jp2c(self): """jp2h must precede jp2c""" j2k = Jp2k(self.j2kfile) From 09651c27b63901d1f1eb5419e458f6950935ee10 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 10 Feb 2014 19:38:37 -0500 Subject: [PATCH 41/83] No longer claiming any windows support. #167 --- docs/source/detailed_installation.rst | 22 +++++----------------- docs/source/introduction.rst | 8 +++----- docs/source/roadmap.rst | 4 +--- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index 4e17c4f..d1b3084 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -45,13 +45,6 @@ the path will be :: $XDG_CONFIG_HOME/glymur/glymurrc -On windows, the path to the configuration file can be determined -by starting up Python and typing :: - - import os - os.path.join(os.path.expanduser('~'), 'glymur', 'glymurrc') - - You may also include a line for the version 1.x openjpeg library if you have it installed in a non-standard place, i.e. :: @@ -70,9 +63,8 @@ packages/RPMs/ports/whatever without going through pip. Mac OS X -------- -All the necessary packages are available to use glymur with Python 2.6, 2.7, -and 3.3 via MacPorts. For python 3.3, you should install the following set of -ports: +All the necessary packages are available to use glymur with MacPorts. +For python 3.3, you should install the following set of ports: * python33 * py33-numpy @@ -108,12 +100,8 @@ difficulties: Windows ------- -32-bit WinPython 2.7.5 seemed to work with OpenJPEG 1.X, 2.0, and the -development version, but still required contextlib2 and mock to be -installed via pip. WinPython 3.3.2, however, seems to have trouble -with OpenJPEG 2.0, so I would suggest using the development version with -that configuration. I no longer have any access to a windows machine, -so I cannot currently offer much guidance here. +The 0.6.x series of Glymur is untested on windows and I make no promises here. +I suggest that windows users check the 0.5.x series. ''''''' @@ -145,5 +133,5 @@ OpenJPEG counterparts are already failing, and others which do pass but still produce heaps of output on stderr. Rather than let this swamp the signal (that most of those tests are actually passing), they've been filtered out for now. There are also more skipped tests on Python 2.7 -than on Python 3.3. The important part is whether or not any test +than on Python3. The important part is whether or not any test errors are reported at the end. diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index 23698e3..38193eb 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -11,8 +11,8 @@ In regards to metadata, most JP2 boxes are properly interpreted. Certain optional JP2 boxes can also be written, including XML boxes and XMP UUIDs. There is some very limited support for reading JPX metadata. -Glymur 0.6 works on Python versions 2.7, 3.3 and 3.4. If you have 2.6, you should -use the 0.5 series. +Glymur 0.6 works on Python versions 2.7, 3.3 and 3.4. If you have Python 2.6, +you should use the 0.5 series of Glymur. OpenJPEG Installation ===================== @@ -24,9 +24,7 @@ OpenJPEG, please consult http://www.openjpeg.org. If you use MacPorts or if you have a sufficiently recent version of Linux, your package manager should already provide you with a version of -OpenJPEG 1.X which glymur can already use. If your platform is windows, -I suggest using the windows installers provided to you by the OpenJPEG -folks at https://code.google.com/p/openjpeg/downloads/list . +OpenJPEG 1.X which glymur can already use. Glymur Installation =================== diff --git a/docs/source/roadmap.rst b/docs/source/roadmap.rst index c29d4f4..6651587 100644 --- a/docs/source/roadmap.rst +++ b/docs/source/roadmap.rst @@ -5,6 +5,4 @@ Roadmap Here's an incomplete list of what I'd like to focus on in the future. * continue to monitor upstream changes in the openjp2 library - * investigate using CFFI or cython instead of ctypes to wrap openjp2 - * investigate adding write support for UUID/XMP boxes (potentially a big project) - * investigate JPIP (likely to be an even bigger project) + * investigate JPIP (likely to be a big project) From 0feca733992eacdb86cdd084984b78a044d705a1 Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 11 Feb 2014 08:23:03 -0500 Subject: [PATCH 42/83] Added changelog section to docs. #171 --- docs/source/changelog.rst | 42 +++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 43 insertions(+) create mode 100644 docs/source/changelog.rst diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..f7e5921 --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,42 @@ +--------- +ChangeLog +--------- + +0.6.0 (pending) +=============== + + * added set_printoptions, get_printoptions function + * dropped support for Python 2.6, added support for Python 3.4 + * dropped windows support + * added write support for JP2 UUID, dataEntryURL, palette, and component mapping boxes + * added read/write support for JPX free, number list, and data reference boxes + * Added read support for JPX fragment list and fragment table boxes + * incompatible change to palette box constructor, it now takes a 2D numpy array instead of a list of 1D arrays + +0.5.0 (September 16, 2013) +========================== + + * added write support when using OpenJPEG version 1.5 + * added version module + +0.4.0 (August 18, 2013) +========================== + + * added append method + +0.3.0 (July 31, 2013) +========================== + + * added support for OpenJPEG library version 2.0.0 + +0.2.0 (July 11, 2013) +========================== + + * added Python 2.6, Python 2.7 on windows + * read/write using OpenJPEG library version 2.0.0 + * read using OpenJPEG 1.4 + +0.1.0 (May 27, 2013) +==================== + + * first release using development version (2.x) of OpenJPEG diff --git a/docs/source/index.rst b/docs/source/index.rst index 8536be8..792f75a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,6 +16,7 @@ Contents: detailed_installation how_do_i roadmap + changelog api ------------------ From df5bf539eff559c27749d04649f04ca195761108 Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 11 Feb 2014 13:39:56 -0500 Subject: [PATCH 43/83] Some refactoring to reduce complexity for box validity. #170 --- glymur/jp2k.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 55e3766..43e24b4 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1122,6 +1122,11 @@ def _validate_jp2_box_sequence(boxes): msg += "must be the file type box." raise IOError(msg) + # The compatibility list must contain at a minimum 'jp2 '. + if 'jp2 ' not in boxes[1].compatibility_list: + msg = "The ftyp box must contain 'jp2 ' in the compatibility list." + raise IOError(msg) + # jp2c must be preceeded by jp2h jp2h_lst = [idx for (idx, box) in enumerate(boxes) if box.box_id == 'jp2h'] @@ -1138,14 +1143,22 @@ def _validate_jp2_box_sequence(boxes): msg = "The codestream box must be preceeded by a jp2 header box." raise IOError(msg) + _validate_jp2h(boxes[jp2h_idx]) + _check_jp2h_child_boxes(boxes, 'top-level') + _asoc_check(boxes) + _jpx_brand(boxes, boxes[1].brand) + _jpx_compatibility(boxes, boxes[1].compatibility_list) + _check_for_singletons(boxes) + _check_top_level(boxes) + +def _validate_jp2h(jp2h): + """Validate the JP2 Header box.""" # 1st jp2 header box cannot be empty. - jp2h = boxes[jp2h_idx] if len(jp2h.box) == 0: msg = "The JP2 header superbox cannot be empty." raise IOError(msg) # 1st jp2 header box must be ihdr - jp2h = boxes[jp2h_idx] if jp2h.box[0].box_id != 'ihdr': msg = "The first box in the jp2 header box must be the image " msg += "header box." @@ -1159,15 +1172,12 @@ def _validate_jp2_box_sequence(boxes): raise IOError(msg) colr = jp2h.box[colr_lst[0]] - # Any cdef box must be in the jp2 header following the image header. - cdef_lst = [j for (j, box) in enumerate(boxes) if box.box_id == 'cdef'] - if len(cdef_lst) != 0: - msg = "Any channel defintion box must be in the JP2 header " - msg += "following the image header." - raise IOError(msg) + _validate_channel_definition(jp2h, colr) - cdef_lst = [j for (j, box) in enumerate(jp2h.box) - if box.box_id == 'cdef'] + +def _validate_channel_definition(jp2h, colr): + """Validate the channel definition box.""" + cdef_lst = [j for (j, box) in enumerate(jp2h.box) if box.box_id == 'cdef'] if len(cdef_lst) > 1: msg = "Only one channel definition box is allowed in the " msg += "JP2 header." @@ -1187,18 +1197,6 @@ def _validate_jp2_box_sequence(boxes): msg += "channel definition box." raise IOError(msg) - # The compatibility list must contain at a minimum 'jp2 '. - if 'jp2 ' not in boxes[1].compatibility_list: - msg = "The ftyp box must contain 'jp2 ' in the compatibility list." - raise IOError(msg) - - # JPX checks. - _asoc_check(boxes) - _jpx_brand(boxes, boxes[1].brand) - _jpx_compatibility(boxes, boxes[1].compatibility_list) - _check_for_singletons(boxes) - _check_top_level(boxes) - _check_jp2h_child_boxes(boxes, 'top-level') JP2H_CHILDREN = set(['bpcc', 'cmap', 'ihdr', 'pclr']) def _check_jp2h_child_boxes(boxes, parent_box_name): From 0205a3cbbaa32d919937cb89cd4a9fa98de9cef9 Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 11 Feb 2014 13:57:30 -0500 Subject: [PATCH 44/83] More refactoring, think it's in good shape now. #170 --- glymur/jp2k.py | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 43e24b4..cd1a6eb 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1115,6 +1115,18 @@ def _validate_jp2_box_sequence(boxes): This is non-exhaustive. """ + _validate_signature_compatibility(boxes) + _validate_jp2h(boxes) + _validate_jp2c(boxes) + _validate_association(boxes) + _validate_jpx_brand(boxes, boxes[1].brand) + _validate_jpx_compatibility(boxes, boxes[1].compatibility_list) + _validate_singletons(boxes) + _validate_top_level(boxes) + + +def _validate_signature_compatibility(boxes): + """Validate the file signature and compatibility status.""" # Check for a bad sequence of boxes. # 1st two boxes must be 'jP ' and 'ftyp' if boxes[0].box_id != 'jP ' or boxes[1].box_id != 'ftyp': @@ -1127,6 +1139,9 @@ def _validate_jp2_box_sequence(boxes): msg = "The ftyp box must contain 'jp2 ' in the compatibility list." raise IOError(msg) + +def _validate_jp2c(boxes): + """Validate the codestream box in relation to other boxes.""" # jp2c must be preceeded by jp2h jp2h_lst = [idx for (idx, box) in enumerate(boxes) if box.box_id == 'jp2h'] @@ -1143,16 +1158,14 @@ def _validate_jp2_box_sequence(boxes): msg = "The codestream box must be preceeded by a jp2 header box." raise IOError(msg) - _validate_jp2h(boxes[jp2h_idx]) - _check_jp2h_child_boxes(boxes, 'top-level') - _asoc_check(boxes) - _jpx_brand(boxes, boxes[1].brand) - _jpx_compatibility(boxes, boxes[1].compatibility_list) - _check_for_singletons(boxes) - _check_top_level(boxes) -def _validate_jp2h(jp2h): +def _validate_jp2h(boxes): """Validate the JP2 Header box.""" + _check_jp2h_child_boxes(boxes, 'top-level') + + jp2h_lst = [box for box in boxes if box.box_id == 'jp2h'] + jp2h = jp2h_lst[0] + # 1st jp2 header box cannot be empty. if len(jp2h.box) == 0: msg = "The JP2 header superbox cannot be empty." @@ -1196,7 +1209,7 @@ def _validate_channel_definition(jp2h, colr): msg = "All color channels must be defined in the " msg += "channel definition box." raise IOError(msg) - + JP2H_CHILDREN = set(['bpcc', 'cmap', 'ihdr', 'pclr']) def _check_jp2h_child_boxes(boxes, parent_box_name): @@ -1241,7 +1254,7 @@ def _check_superbox_for_top_levels(boxes): if hasattr(box, 'box'): _check_superbox_for_top_levels(box.box) -def _check_top_level(boxes): +def _validate_top_level(boxes): """Several boxes can only occur at the top level.""" # Add the counts in the superboxes. for box in boxes: @@ -1254,7 +1267,7 @@ def _check_top_level(boxes): if 'dtbl' in multiples: raise IOError('There can only be one dtbl box in a file.') -def _check_for_singletons(boxes): +def _validate_singletons(boxes): """Several boxes can only occur once.""" count = _collect_box_count(boxes) # Which boxes occur more than once? @@ -1262,7 +1275,7 @@ def _check_for_singletons(boxes): if 'dtbl' in multiples: raise IOError('There can only be one dtbl box in a file.') -def _jpx_brand(boxes, brand): +def _validate_jpx_brand(boxes, brand): """ If there is a JPX box then the brand must be 'jpx '. """ @@ -1274,9 +1287,9 @@ def _jpx_brand(boxes, brand): raise RuntimeError(msg) if hasattr(box, 'box') != 0: # Same set of checks on any child boxes. - _jpx_brand(box.box, brand) + _validate_jpx_brand(box.box, brand) -def _jpx_compatibility(boxes, compatibility_list): +def _validate_jpx_compatibility(boxes, compatibility_list): """ If there is a JPX box then the compatibility list must also contain 'jpx '. """ @@ -1288,10 +1301,10 @@ def _jpx_compatibility(boxes, compatibility_list): raise RuntimeError(msg) if hasattr(box, 'box') != 0: # Same set of checks on any child boxes. - _jpx_compatibility(box.box, compatibility_list) + _validate_jpx_compatibility(box.box, compatibility_list) -def _asoc_check(boxes): +def _validate_association(boxes): """ Association boxes can only contain number list boxes and xml boxes, as far as we know. @@ -1304,7 +1317,7 @@ def _asoc_check(boxes): raise RuntimeError(msg) if hasattr(box, 'box') != 0: # Same set of checks on any child boxes. - _asoc_check(box.box) + _validate_association(box.box) From b0719450b5bb8f3517956fbc285e5a93f841aacb Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 11 Feb 2014 16:16:22 -0500 Subject: [PATCH 45/83] Added support for rreq boxes with mask length of 3. #165 --- glymur/jp2box.py | 68 ++++++++++++++++++++++++++++++++-- glymur/test/test_jp2box.py | 5 +-- glymur/test/test_jp2box_jpx.py | 34 ++++++----------- glymur/test/test_jp2k.py | 4 +- 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 07ddb6e..a6ea821 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1837,10 +1837,10 @@ class ReaderRequirementsBox(Jp2kBox): Jp2kBox.__init__(self, box_id='rreq', longname='Reader Requirements') self.fuam = fuam self.dcm = dcm - self.standard_flag = standard_flag - self.standard_mask = standard_mask - self.vendor_feature = vendor_feature - self.vendor_mask = vendor_mask + self.standard_flag = tuple(standard_flag) + self.standard_mask = tuple(standard_mask) + self.vendor_feature = tuple(vendor_feature) + self.vendor_mask = tuple(vendor_mask) self.length = length self.offset = offset @@ -1902,6 +1902,9 @@ class ReaderRequirementsBox(Jp2kBox): read_buffer = fptr.read(1) mask_length, = struct.unpack('>B', read_buffer) + if mask_length == 3: + return _parse_rreq3(fptr, length, offset) + # Fully Understands Aspect Mask # Decodes Completely Mask read_buffer = fptr.read(2 * mask_length) @@ -1911,6 +1914,7 @@ class ReaderRequirementsBox(Jp2kBox): # The mask length tells us the format string to use when unpacking # from the buffer read from file. + try: mask_format = {1: 'B', 2: 'H', 4: 'I', 8: 'Q'}[mask_length] fuam, dcm = struct.unpack('>' + mask_format * 2, read_buffer) @@ -1931,6 +1935,62 @@ class ReaderRequirementsBox(Jp2kBox): return box +def _parse_rreq3(fptr, length, offset): + """Parse a reader requirements box. Special case when mask length is 3.""" + # Fully Understands Aspect Mask + # Decodes Completely Mask + read_buffer = fptr.read(2 * 3) + + fuam = dcm = standard_flag = standard_mask = [] + vendor_feature = vendor_mask = [] + + # The mask length tells us the format string to use when unpacking + # from the buffer read from file. + lst = struct.unpack('>BBBBBB', read_buffer) + fuam = lst[0] << 16 | lst[1] << 8 | lst[2] + dcm = lst[3] << 16 | lst[4] << 8 | lst[5] + + read_buffer = fptr.read(2) + num_standard_features, = struct.unpack('>H', read_buffer) + + fmt = '>' + 'HBBB' * num_standard_features + read_buffer = fptr.read(num_standard_features * 5) + lst = struct.unpack(fmt, read_buffer) + + standard_flag = lst[0::4] + standard_mask = [] + for j in range(num_standard_features): + items = lst[slice(j * 4 + 1, j * 4 + 4)] + mask = items[0] << 16 | items[1] << 8 | items[2] + standard_mask.append(mask) + + read_buffer = fptr.read(2) + num_vendor_features, = struct.unpack('>H', read_buffer) + + fmt = '>' + 'HBBB' * num_vendor_features + read_buffer = fptr.read(num_vendor_features * 5) + lst = struct.unpack(fmt, read_buffer) + + # Each vendor feature consists of a 16-byte UUID plus a mask whose + # length is specified by, you guessed it, "mask_length". + entry_length = 16 + 3 + read_buffer = fptr.read(num_vendor_features * entry_length) + vendor_feature = [] + vendor_mask = [] + for j in range(num_vendor_features): + ubuffer = read_buffer[j * entry_length:(j + 1) * entry_length] + vendor_feature.append(uuid.UUID(bytes=ubuffer[0:16])) + + lst = struct.unpack('>BBB', ubuffer[16:]) + vmask = lst[0] << 16 | lst[1] << 8 | lst[2] + vendor_mask.append(vmask) + + box = ReaderRequirementsBox(fuam, dcm, standard_flag, standard_mask, + vendor_feature, vendor_mask, + length=length, offset=offset) + return box + + def _parse_standard_flag(fptr, mask_length): """Construct standard flag, standard mask data from the file. diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index ac1b2a3..ea1fddb 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -580,10 +580,7 @@ class TestWrap(unittest.TestCase): def test_jpx_to_jp2(self): """basic test for rewrapping a jpx file""" - with warnings.catch_warnings(): - # This file has a rreq mask length that we do not recognize. - warnings.simplefilter("ignore") - jpx = Jp2k(self.jpxfile) + jpx = Jp2k(self.jpxfile) idx = [0, 1, 3, 6] boxes = [jpx.box[idx] for idx in [0, 1, 3, 6]] with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 571afbc..93b821f 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -135,7 +135,6 @@ class TestJPXWrap(unittest.TestCase): jpx = jp2.wrap(tfile.name, boxes=boxes) -@unittest.skipIf(sys.hexversion < 0x03000000, "Warning assert on 2.x.") @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") class TestJPX(unittest.TestCase): """Test suite for other JPX boxes.""" @@ -146,22 +145,18 @@ class TestJPX(unittest.TestCase): def tearDown(self): pass - def test_rreq_box_strange_mask_length(self): - """The standard says that the mask length should be 1, 2, 4, or 8.""" - with warnings.catch_warnings(): - # This file has a rreq mask length that we do not recognize. - warnings.simplefilter("ignore") - j = Jp2k(self.jpxfile) - self.assertEqual(j.box[2].box_id, 'rreq') - self.assertEqual(type(j.box[2]), + def test_jpx_rreq_mask_length_3(self): + """There are some JPX files with rreq mask length of 3.""" + jpx = Jp2k(self.jpxfile) + self.assertEqual(jpx.box[2].box_id, 'rreq') + self.assertEqual(type(jpx.box[2]), glymur.jp2box.ReaderRequirementsBox) + self.assertEqual(jpx.box[2].standard_flag, + (5, 42, 45, 2, 18, 19, 1, 8, 12, 31, 20)) def test_free_box(self): """Verify that we can handle a free box.""" - with warnings.catch_warnings(): - # This file has a rreq mask length that we do not recognize. - warnings.simplefilter("ignore") - j = Jp2k(self.jpxfile) + j = Jp2k(self.jpxfile) self.assertEqual(j.box[16].box[0].box_id, 'free') self.assertEqual(type(j.box[16].box[0]), glymur.jp2box.FreeBox) @@ -183,10 +178,7 @@ class TestJPX(unittest.TestCase): tfile.flush() - with warnings.catch_warnings(): - # This file has a rreq mask length that we do not recognize. - warnings.simplefilter("ignore") - jpx = Jp2k(tfile.name) + jpx = Jp2k(tfile.name) self.assertEqual(jpx.box[-1].box_id, 'dtbl') self.assertEqual(len(jpx.box[-1].DR), 2) @@ -212,8 +204,7 @@ class TestJPX(unittest.TestCase): tfile.flush() - with self.assertWarns(UserWarning): - jpx = Jp2k(tfile.name) + jpx = Jp2k(tfile.name) self.assertEqual(jpx.box[-1].box_id, 'ftbl') self.assertEqual(jpx.box[-1].box[0].box_id, 'flst') @@ -223,10 +214,7 @@ class TestJPX(unittest.TestCase): def test_nlst(self): """Verify that we can handle a free box.""" - with warnings.catch_warnings(): - # This file has a rreq mask length that we do not recognize. - warnings.simplefilter("ignore") - j = Jp2k(self.jpxfile) + j = Jp2k(self.jpxfile) self.assertEqual(j.box[16].box[1].box[0].box_id, 'nlst') self.assertEqual(type(j.box[16].box[1].box[0]), glymur.jp2box.NumberListBox) diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index a36d42c..386f077 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -379,9 +379,7 @@ class TestJp2k(unittest.TestCase): def test_jpx_mult_codestreams_jp2_brand(self): """Read JPX codestream when jp2-compatible.""" # The file in question has multiple codestreams. - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - jpx = Jp2k(self.jpxfile) + jpx = Jp2k(self.jpxfile) data = jpx.read() if re.match(r"""1\.[0123]""", glymur.version.openjpeg_version): # openjpeg 1.3 doesn't apply the palette, so it's a 2D image here From e2539b04e12289cbfb2f9dbc44abd06ba139028c Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 11 Feb 2014 19:59:35 -0500 Subject: [PATCH 46/83] Added deprecated rreq descriptions. #173 --- glymur/jp2box.py | 49 +++++++++++++++++++++----------------- glymur/jp2k.py | 2 +- glymur/test/fixtures.py | 2 +- glymur/test/test_jp2box.py | 2 +- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index a6ea821..03adce0 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1721,19 +1721,21 @@ class PaletteBox(Jp2kBox): # Map rreq codes to display text. _READER_REQUIREMENTS_DISPLAY = { 0: 'File not completely understood', - 1: 'Deprecated', + 1: 'Deprecated - contains no extensions', 2: 'Contains multiple composition layers', - 3: 'Deprecated', + 3: 'Deprecated - codestream is compressed using JPEG 2000 and requires ' + + 'at least a Profile 0 decoder as defind in ITU-T Rec. T.800 ' + + '| ISO/IEC 15444-1, A.10 Table A.45', 4: 'JPEG 2000 Part 1 Profile 1 codestream', 5: 'Unrestricted JPEG 2000 Part 1 codestream, ITU-T Rec. T.800 ' + '| ISO/IEC 15444-1', 6: 'Unrestricted JPEG 2000 Part 2 codestream', 7: 'JPEG codestream as defined in ISO/IEC 10918-1', - 8: 'Deprecated', + 8: 'Deprecated - does not contain opacity', 9: 'Non-premultiplied opacity channel', 10: 'Premultiplied opacity channel', 11: 'Chroma-key based opacity', - 12: 'Deprecated', + 12: 'Deprecated - codestream is contiguous', 13: 'Fragmented codestream where all fragments are in file and in order', 14: 'Fragmented codestream where all fragments are in file ' + 'but are out of order', @@ -1743,21 +1745,24 @@ _READER_REQUIREMENTS_DISPLAY = { + 'only through a URL specified network connection', 17: 'Compositing required to produce rendered result from multiple ' + 'compositing layers', - 18: 'Deprecated', - 19: 'Deprecated', - 20: 'Deprecated', + 18: 'Deprecated - support for compositing is not required', + 19: 'Deprecated - contains multiple, discrete layers that should not ' + + 'be combined through either animation or compositing', + 20: 'Deprecated - compositing layers each contain only a single ' + + 'codestream', 21: 'At least one compositing layer consists of multiple codestreams', - 22: 'Deprecated', + 22: 'Deprecated - all compositing layers are in the same colourspace', 23: 'Colourspace transformations are required to combine compositing ' + 'layers; not all compositing layers are in the same colourspace', - 24: 'Deprecated', - 25: 'Deprecated', + 24: 'Deprecated - rendered result created without using animation', + 25: 'Deprecated - animated, but first layer covers entire area and is ' + + 'opaque', 26: 'First animation layer does not cover entire rendered result', - 27: 'Deprecated', + 27: 'Deprecated - animated, and no layer is reused', 28: 'Reuse of animation layers', - 29: 'Deprecated', + 29: 'Deprecated - animated, but layers are reused', 30: 'Some animated frames are non-persistent', - 31: 'Deprecated', + 31: 'Deprecated - rendered result created without using scaling', 32: 'Rendered result involves scaling within a layer', 33: 'Rendered result involves scaling between layers', 34: 'ROI metadata', @@ -1768,11 +1773,11 @@ _READER_REQUIREMENTS_DISPLAY = { 39: 'JPX digital signatures', 40: 'JPX checksums', 41: 'Desires Graphics Arts Reproduction specified', - 42: 'Deprecated', - 43: '(Deprecated) compositing layer uses restricted ICC profile', + 42: 'Deprecated - compositing layer uses palettized colour', + 43: 'Deprecated - compositing layer uses restricted ICC profile', 44: 'Compositing layer uses Any ICC profile', - 45: 'Deprecated', - 46: 'Deprecated', + 45: 'Deprecated - compositing layer uses sRGB enumerated colourspace', + 46: 'Deprecated - compositing layer uses sRGB-grey enumerated colourspace', 47: 'BiLevel 1 enumerated colourspace', 48: 'BiLevel 2 enumerated colourspace', 49: 'YCbCr 1 enumerated colourspace', @@ -1789,14 +1794,14 @@ _READER_REQUIREMENTS_DISPLAY = { 60: 'e-sRGB enumerated colorspace', 61: 'ROMM_RGB enumerated colorspace', 62: 'Non-square samples', - 63: 'Deprecated', - 64: 'Deprecated', - 65: 'Deprecated', - 66: 'Deprecated', + 63: 'Deprecated - compositing layers have labels', + 64: 'Deprecated - codestreams have labels', + 65: 'Deprecated - compositing layers have different colour spaces', + 66: 'Deprecated - compositing layers have different metadata', 67: 'GIS metadata XML box', 68: 'JPSEC extensions in codestream as specified by ISO/IEC 15444-8', 69: 'JP3D extensions in codestream as specified by ISO/IEC 15444-10', - 70: 'Deprecated', + 70: 'Deprecated - compositing layer uses sYCC enumerated colour space', 71: 'e-sYCC enumerated colourspace', 72: 'JPEG 2000 Part 2 codestream as restricted by baseline conformance ' + 'requirements in M.9.2.3', diff --git a/glymur/jp2k.py b/glymur/jp2k.py index cd1a6eb..fbbf999 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1211,7 +1211,7 @@ def _validate_channel_definition(jp2h, colr): raise IOError(msg) -JP2H_CHILDREN = set(['bpcc', 'cmap', 'ihdr', 'pclr']) +JP2H_CHILDREN = set(['bpcc', 'cdef', 'cmap', 'ihdr', 'pclr']) def _check_jp2h_child_boxes(boxes, parent_box_name): """Certain boxes can only reside in the JP2 header.""" box_ids = set([box.box_id for box in boxes]) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 5f7d77a..9b40a25 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -692,5 +692,5 @@ file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) Standard Features: Feature 005: Unrestricted JPEG 2000 Part 1 codestream, ITU-T Rec. T.800 | ISO/IEC 15444-1 Feature 060: e-sRGB enumerated colorspace - Feature 043: (Deprecated) compositing layer uses restricted ICC profile + Feature 043: Deprecated - compositing layer uses restricted ICC profile Vendor Features:""" diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index ea1fddb..feffec9 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -315,7 +315,7 @@ class TestChannelDefinition(unittest.TestCase): boxes = [self.jp2b, self.ftyp, self.jp2h, cdef, self.jp2c] with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: - with self.assertRaises(IOError): + with self.assertRaises((IOError, OSError)): j2k.wrap(tfile.name, boxes=boxes) def test_bad_type(self): From db92f3fea774a425bc5f85402bdf4fc6ee81c177 Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 12 Feb 2014 09:17:17 -0500 Subject: [PATCH 47/83] rreq masks and standard features now printed together. #174 Masks are printed in hex. --- glymur/jp2box.py | 18 ++++++------------ glymur/test/fixtures.py | 14 +++++--------- glymur/test/test_printing.py | 6 +++--- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 03adce0..d5e79e1 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -11,7 +11,7 @@ References Extensions """ -# pylint: disable=C0302,R0903,R0913 +# pylint: disable=C0302,R0903,R0913,W0142 from collections import OrderedDict import datetime @@ -1866,20 +1866,14 @@ class ReaderRequirementsBox(Jp2kBox): if _printoptions['short'] == True: return msg - msg += '\n Fully Understands Aspect Mask: {0}'.format(self.fuam) - msg += '\n Display Completely Mask: {0}'.format(self.dcm) + msg += '\n Fully Understands Aspect Mask: 0x{0:x}'.format(self.fuam) + msg += '\n Display Completely Mask: 0x{0:x}'.format(self.dcm) msg += '\n Standard Features and Masks:' for j in range(len(self.standard_flag)): - sfl = self.standard_flag[j] - mask = self.standard_mask[j] - msg += '\n Feature {0:03d}: {1}'.format(sfl, mask) - - msg += '\n Standard Features:' - for j in range(len(self.standard_flag)): - sfl = self.standard_flag[j] - rrdisp = _READER_REQUIREMENTS_DISPLAY[self.standard_flag[j]] - msg += '\n Feature {0:03d}: {1}'.format(sfl, rrdisp) + args = (self.standard_flag[j], self.standard_mask[j], + _READER_REQUIREMENTS_DISPLAY[self.standard_flag[j]]) + msg += '\n Feature {0:03d}: 0x{1:x} {2}'.format(*args) msg += '\n Vendor Features:' for j in range(len(self.vendor_feature)): diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 9b40a25..6e0a76c 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -683,14 +683,10 @@ Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" # Output of reader requirement printing for file7.jp2 file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) - Fully Understands Aspect Mask: 160 - Display Completely Mask: 192 + Fully Understands Aspect Mask: 0xa0 + Display Completely Mask: 0xc0 Standard Features and Masks: - Feature 005: 128 - Feature 060: 96 - Feature 043: 64 - Standard Features: - Feature 005: Unrestricted JPEG 2000 Part 1 codestream, ITU-T Rec. T.800 | ISO/IEC 15444-1 - Feature 060: e-sRGB enumerated colorspace - Feature 043: Deprecated - compositing layer uses restricted ICC profile + Feature 005: 0x80 Unrestricted JPEG 2000 Part 1 codestream, ITU-T Rec. T.800 | ISO/IEC 15444-1 + Feature 060: 0x60 e-sRGB enumerated colorspace + Feature 043: 0x40 Deprecated - compositing layer uses restricted ICC profile Vendor Features:""" diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 26156dc..5fe3322 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -53,7 +53,7 @@ class TestPrinting(unittest.TestCase): with self.assertRaises(TypeError): glymur.set_printoptions(hi='low') - def test_printopt_no_codestr_then_no_xml(self): + def test_propts_no_codestream_then_no_xml(self): """Verify printed output when codestream=False and xml=False, #162""" # The print options should be persistent across invocations. glymur.set_printoptions(codestream=False) @@ -629,7 +629,7 @@ class TestPrinting(unittest.TestCase): def test_xml_latin1(self): """Should be able to print an XMLBox with utf-8 encoding (latin1).""" # Seems to be inconsistencies between different versions of python2.x - # as to what gets printed. + # as to what gets printed. # # 2.7.5 (fedora 19) prints xml entities. # 2.7.3 seems to want to print hex escapes. @@ -658,7 +658,7 @@ class TestPrinting(unittest.TestCase): def test_xml_cyrrilic(self): """Should be able to print an XMLBox with utf-8 encoding (cyrrillic).""" # Seems to be inconsistencies between different versions of python2.x - # as to what gets printed. + # as to what gets printed. # # 2.7.5 (fedora 19) prints xml entities. # 2.7.3 seems to want to print hex escapes. From 1906bb60041dfcfe7a2adadf5bc48c3be34194e2 Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 12 Feb 2014 10:00:35 -0500 Subject: [PATCH 48/83] Added 'jpxb' to accepted items in the ftyp compatibility list. #172 --- glymur/jp2k.py | 7 ++++--- glymur/test/test_jp2box_jpx.py | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index fbbf999..d159fcc 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1293,11 +1293,12 @@ def _validate_jpx_compatibility(boxes, compatibility_list): """ If there is a JPX box then the compatibility list must also contain 'jpx '. """ + jpx_cl = set(compatibility_list) for box in boxes: if box.box_id in JPX_IDS: - if 'jpx ' not in compatibility_list: - msg = "A JPX box requires that 'jpx ' be present in the " - msg += "ftype compatibility list." + if len(set(['jpx ', 'jpxb']).intersection(jpx_cl)) == 0: + msg = "A JPX box requires that either 'jpx ' or 'jpxb' be " + msg += "present in the ftype compatibility list." raise RuntimeError(msg) if hasattr(box, 'box') != 0: # Same set of checks on any child boxes. diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 93b821f..2ebeb1f 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -40,6 +40,33 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_jpxb_compatibility(self): + """Wrap JP2 to JPX, state jpxb compatibility""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + # The ftyp box must be modified to jpx with jp2 compatibility. + boxes[1].brand = 'jpx ' + boxes[1].compatibility_list = ['jp2 ', 'jpxb'] + + numbers = (0, 1) + nlst = glymur.jp2box.NumberListBox(numbers) + the_xml = ET.fromstring('0') + xmlb = glymur.jp2box.XMLBox(xml=the_xml) + asoc = glymur.jp2box.AssociationBox([nlst, xmlb]) + boxes.append(asoc) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + jpx = jp2.wrap(tfile.name, boxes=boxes) + + self.assertEqual(jpx.box[1].compatibility_list, ['jp2 ', 'jpxb']) + self.assertEqual(jpx.box[-1].box_id, 'asoc') + self.assertEqual(jpx.box[-1].box[0].box_id, 'nlst') + self.assertEqual(jpx.box[-1].box[1].box_id, 'xml ') + self.assertEqual(jpx.box[-1].box[0].associations, numbers) + self.assertEqual(ET.tostring(jpx.box[-1].box[1].xml.getroot()), + b'0') + def test_association_box(self): """Wrap JP2 to JPX with asoc(nlst, xml)""" jp2 = Jp2k(self.jp2file) From d016fc504abd4f160124737dde68a0b1ee6fbc4f Mon Sep 17 00:00:00 2001 From: jevans Date: Wed, 12 Feb 2014 19:35:45 -0500 Subject: [PATCH 49/83] Added another XMP example. #169 --- docs/source/how_do_i.rst | 73 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index dde5b60..a1fe089 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -5,8 +5,9 @@ How do I...? ... read the lowest resolution thumbnail? ========================================= -Printing the Jp2k object should reveal the number of resolutions (look in the -COD segment section), but you can take a shortcut by supplying -1 as the +Printing the Jp2k object should reveal the number of resolutions +(look in the COD segment section of the codestream), but you can +take a shortcut by supplying -1 as the resolution level. :: >>> import glymur @@ -381,6 +382,8 @@ Here's how the Preview application on the mac shows the RGBA image. ... work with XMP UUIDs? ======================== +XMP is metadata on steroids. + The example JP2 file shipped with glymur has an XMP UUID. :: >>> import glymur @@ -518,3 +521,69 @@ though, you have to know the UUID that signifies XMP data.:: + +You can also build up XMP metadata from scratch. For instance, if we try to +wrap `goodstuff.j2k` again:: + + >>> import glymur + >>> jfile = glymur.data.goodstuff() + >>> j2k = glymur.Jp2k(jfile) + >>> jp2 = j2k.wrap("goodstuff.jp2") + +Now build up the metadata piece-by-piece. It would help to have the XMP +standard close at hand:: + + >>> from libxmp import XMPMeta + >>> from libxmp.consts import XMP_NS_TIFF as NS_TIFF + >>> from libxmp.consts import XMP_NS_DC as NS_DC + >>> xmp = XMPMeta() + >>> ihdr = jp2.box[2].box[0] + >>> xmp.set_property(NS_TIFF, "ImageWidth", str(ihdr.width)) + >>> xmp.set_property(NS_TIFF, "ImageHeight", str(ihdr.height)) + >>> xmp.set_property(NS_TIFF, "BitsPerSample", '3') + >>> xmp.set_property(NS_DC, "Title", u'Stürm und Drang') + >>> xmp.set_property(NS_DC, "Creator", 'Glymur') + +We can then append the XMP in a UUID box just as before:: + + >>> import uuid + >>> xmp_uuid = uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac') + >>> box = glymur.jp2box.UUIDBox(xmp_uuid, str(xmp).encode()) + >>> jp2.append(box) + >>> glymur.set_printoptions(codestream=False) + >>> print(jp2) + File: goodstuff.jp2 + JPEG 2000 Signature Box (jP ) @ (0, 12) + Signature: 0d0a870a + File Type Box (ftyp) @ (12, 20) + Brand: jp2 + Compatibility: ['jp2 '] + JP2 Header Box (jp2h) @ (32, 45) + Image Header Box (ihdr) @ (40, 22) + Size: [800 480 3] + Bitdepth: 8 + Signed: False + Compression: wavelet + Colorspace Unknown: False + Colour Specification Box (colr) @ (62, 15) + Method: enumerated colorspace + Precedence: 0 + Colorspace: sRGB + Contiguous Codestream Box (jp2c) @ (77, 115228) + UUID Box (uuid) @ (115305, 671) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: + + + + 480 + 800 + 3 + + + Stürm und Drang + Glymur + + + + From f264bb00f3cbfe157a8b77ea756275027bc8ddd2 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 13 Feb 2014 11:30:04 -0500 Subject: [PATCH 50/83] Default value for codestream argument should be [0], not 0. --- bin/jp2dump | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/jp2dump b/bin/jp2dump index 08907e5..7632aff 100755 --- a/bin/jp2dump +++ b/bin/jp2dump @@ -15,7 +15,7 @@ parser.add_argument('-c', '--codestream', help=chelp, nargs=1, type=int, - default=0) + default=[0]) parser.add_argument('filename') args = parser.parse_args() if args.noxml: From 5ba6fd5d536909d4f338c328d860fcb6a2b683bb Mon Sep 17 00:00:00 2001 From: jevans Date: Fri, 14 Feb 2014 21:45:40 -0500 Subject: [PATCH 51/83] Added write support for Label box. #168 --- glymur/jp2box.py | 8 ++++++++ glymur/jp2k.py | 16 ++++++++++++++++ glymur/test/test_jp2box_jpx.py | 29 +++++++++++++++++++++++++---- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index d5e79e1..a599f84 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2286,6 +2286,14 @@ class LabelBox(Jp2kBox): msg = 'glymur.jp2box.LabelBox("{0}")'.format(self.label) return msg + def write(self, fptr): + """Write a Label box to file. + """ + length = 8 + len(self.label.encode()) + fptr.write(struct.pack('>I', length)) + fptr.write(self.box_id.encode()) + fptr.write(self.label.encode()) + @staticmethod def parse(fptr, offset, length): """Parse Label box. diff --git a/glymur/jp2k.py b/glymur/jp2k.py index d159fcc..96b6e68 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1119,6 +1119,7 @@ def _validate_jp2_box_sequence(boxes): _validate_jp2h(boxes) _validate_jp2c(boxes) _validate_association(boxes) + _validate_label(boxes) _validate_jpx_brand(boxes, boxes[1].brand) _validate_jpx_compatibility(boxes, boxes[1].compatibility_list) _validate_singletons(boxes) @@ -1304,6 +1305,21 @@ def _validate_jpx_compatibility(boxes, compatibility_list): # Same set of checks on any child boxes. _validate_jpx_compatibility(box.box, compatibility_list) +def _validate_label(boxes): + """ + Label boxes can only be inside association, codestream headers, or + compositing layer header boxes. + """ + for box in boxes: + if box.box_id != 'asoc': + if hasattr(box, 'box'): + for boxi in box.box: + if boxi.box_id == 'lbl ': + msg = "A label box cannot be nested inside a {0} box." + msg = msg.format(box.box_id) + raise IOError(msg) + # Same set of checks on any child boxes. + _validate_label(box.box) def _validate_association(boxes): """ diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 2ebeb1f..8b5e075 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -67,8 +67,8 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(ET.tostring(jpx.box[-1].box[1].xml.getroot()), b'0') - def test_association_box(self): - """Wrap JP2 to JPX with asoc(nlst, xml)""" + def test_association_label_box(self): + """Wrap JP2 to JPX with asoc, label, and nlst boxes""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] @@ -76,11 +76,13 @@ class TestJPXWrap(unittest.TestCase): boxes[1].brand = 'jpx ' boxes[1].compatibility_list = ['jp2 ', 'jpx '] + label = 'this is a test' + lblb = glymur.jp2box.LabelBox(label) numbers = (0, 1) nlst = glymur.jp2box.NumberListBox(numbers) the_xml = ET.fromstring('0') xmlb = glymur.jp2box.XMLBox(xml=the_xml) - asoc = glymur.jp2box.AssociationBox([nlst, xmlb]) + asoc = glymur.jp2box.AssociationBox([nlst, xmlb, lblb]) boxes.append(asoc) with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: @@ -89,10 +91,12 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[1].compatibility_list, ['jp2 ', 'jpx ']) self.assertEqual(jpx.box[-1].box_id, 'asoc') self.assertEqual(jpx.box[-1].box[0].box_id, 'nlst') - self.assertEqual(jpx.box[-1].box[1].box_id, 'xml ') self.assertEqual(jpx.box[-1].box[0].associations, numbers) + self.assertEqual(jpx.box[-1].box[1].box_id, 'xml ') self.assertEqual(ET.tostring(jpx.box[-1].box[1].xml.getroot()), b'0') + self.assertEqual(jpx.box[-1].box[2].box_id, 'lbl ') + self.assertEqual(jpx.box[-1].box[2].label, label) def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" @@ -111,6 +115,23 @@ class TestJPXWrap(unittest.TestCase): with self.assertRaises(IOError): jpx = jp2.wrap(tfile.name, boxes=boxes) + def test_lbl_at_top_level(self): + """Label boxes can only be inside a asoc box .""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + flag = 0 + version = (0, 0, 0) + url = 'file:////usr/local/bin' + lblb = glymur.jp2box.LabelBox('hi there') + + # Put it inside the jp2 header box. + boxes[2].box.append(lblb) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jpx = jp2.wrap(tfile.name, boxes=boxes) + def test_data_reference_not_at_top_level(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) From 75785e4ab067288e58ba0c0335272ac2de563521 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 15 Feb 2014 16:37:08 -0500 Subject: [PATCH 52/83] Refactored __str__ for superboxes. #175 --- glymur/jp2box.py | 83 ++++++++++++------------------------------------ 1 file changed, 20 insertions(+), 63 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index a599f84..e358eab 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -88,6 +88,18 @@ class Jp2kBox(object): msg = "Not supported for {0} box.".format(self.longname) raise NotImplementedError(msg) + def _str_superbox(self): + """__str__ method for all superboxes.""" + msg = Jp2kBox.__str__(self) + for box in self.box: + boxstr = str(box) + + # Add indentation. + strs = [('\n ' + x) for x in boxstr.split('\n')] + msg += ''.join(strs) + return msg + + def _write_superbox(self, fptr): """Write a superbox. @@ -592,16 +604,7 @@ class CodestreamHeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -659,16 +662,7 @@ class CompositingLayerHeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -831,10 +825,10 @@ class ContiguousCodestreamBox(Jp2kBox): msg += '\n Main header:' for segment in self.main_header.segment: segstr = str(segment) - # Add indentation. strs = [('\n ' + x) for x in segstr.split('\n')] msg += ''.join(strs) + return msg @staticmethod @@ -1156,16 +1150,7 @@ class FragmentTableBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -1415,16 +1400,7 @@ class AssociationBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - if _printoptions['short'] == True: - return msg - - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -1485,13 +1461,7 @@ class JP2HeaderBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg def write(self, fptr): @@ -2086,13 +2056,7 @@ class ResolutionBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + msg = self._str_superbox() return msg @staticmethod @@ -2612,14 +2576,7 @@ class UUIDInfoBox(Jp2kBox): return msg def __str__(self): - msg = Jp2kBox.__str__(self) - for box in self.box: - box_str = str(box) - - # Add indentation. - lst = [('\n ' + x) for x in box_str.split('\n')] - msg += ''.join(lst) - + msg = self._str_superbox() return msg @staticmethod From 3c333e999da41cb6129cd9ce346e5b72caf2c7e1 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 20:05:39 -0500 Subject: [PATCH 53/83] Parsing unknown superboxes with known 1st sub box. #175 --- glymur/jp2box.py | 22 +++++++++++++++++++++- glymur/test/test_jp2box_jpx.py | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index e358eab..4810834 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -164,12 +164,32 @@ class Jp2kBox(object): # Call the proper parser for the given box with ID "T". try: box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError: + except KeyError as err: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) box = Jp2kBox(box_id, offset=start, length=num_bytes, longname='Unknown box') + if fptr.tell() != start + 8: + # If the file pointer has advanced, then the KeyError + # ocurred during the parsing of the box. + pass + else: + # Could it be a superbox with recognizable child boxes? + # Peek ahead to see. + pos = fptr.tell() + read_buffer = fptr.read(8) + sub_length, sub_id = struct.unpack('>I4s', read_buffer) + sub_id = sub_id.decode('utf-8') + + # Regardless of whether or not we recognize the box, rewind back + # to properly advance to the next box. + fptr.seek(pos) + + # Now process any child boxes if we actually did recognize it. + if sub_id in _BOX_WITH_ID.keys(): + box.box = box.parse_superbox(fptr) + superbox.append(box) # Position to the start of the next box. diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 8b5e075..6624291 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -4,6 +4,7 @@ Test suite specifically targeting JPX box layout. """ import os +import shutil import struct import sys import tempfile @@ -202,6 +203,25 @@ class TestJPX(unittest.TestCase): self.assertEqual(jpx.box[2].standard_flag, (5, 42, 45, 2, 18, 19, 1, 8, 12, 31, 20)) + @unittest.skipIf(sys.hexversion < 0x03000000, "Needs unittest in 3.x.") + def test_unknown_superbox(self): + """Verify that we can handle an unknown superbox.""" + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with open(self.jpxfile, 'rb') as ifile: + tfile.write(ifile.read()) + + # Add the header for an unknwon superbox. + write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) + tfile.write(write_buffer) + write_buffer = struct.pack('>I4sI', 12, 'free'.encode(), 0) + tfile.write(write_buffer) + tfile.flush() + + with self.assertWarns(UserWarning): + jpx = Jp2k(tfile.name) + self.assertEqual(jpx.box[-1].box_id, 'grp ') + self.assertEqual(jpx.box[-1].box[0].box_id, 'free') + def test_free_box(self): """Verify that we can handle a free box.""" j = Jp2k(self.jpxfile) From 9999a60656592d0b6b613f5010a0f2863d30591a Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 21:43:05 -0500 Subject: [PATCH 54/83] Improved the warning when an exif tag is bad. #175 --- glymur/_uuid_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glymur/_uuid_io.py b/glymur/_uuid_io.py index d4971a1..ed610e5 100644 --- a/glymur/_uuid_io.py +++ b/glymur/_uuid_io.py @@ -146,7 +146,7 @@ class _Ifd(object): tag_name = tagnum2name[tag] except KeyError: # Ok, we don't recognize this tag. Just use the numeric id. - msg = 'Unrecognized Exif tag "{0}".'.format(tag) + msg = 'Unrecognized Exif tag: {0}'.format(tag) warnings.warn(msg, UserWarning) tag_name = tag self.processed_ifd[tag_name] = value From 331f901dae997b6babf9440d48fda26d33daba43 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 22:03:33 -0500 Subject: [PATCH 55/83] Refactored superbox parsing. #175 --- glymur/jp2box.py | 77 ++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 4810834..d2e374b 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -120,6 +120,53 @@ class Jp2kBox(object): fptr.write(struct.pack('>I', end_pos - orig_pos)) fptr.seek(end_pos) + def parse_this_box(self, fptr, box_id, start, num_bytes): + """Parse the current box. + + Parameters + ---------- + fptr : file + Open file object. + box_id : str + 4-letter identifier for the current box. + start, num_bytes: int + Byte offset and length of the current box. + + Returns + ------- + box : Jp2kBox + object corresponding to the current box + """ + try: + box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) + except KeyError as err: + msg = 'Unrecognized box ({0}) encountered.'.format(box_id) + warnings.warn(msg) + box = Jp2kBox(box_id, offset=start, length=num_bytes, + longname='Unknown box') + + if fptr.tell() != start + 8: + # If the file pointer has advanced, then the KeyError + # ocurred during the parsing of the box. + pass + else: + # Could it be a superbox with recognizable child boxes? + # Peek ahead to see. + pos = fptr.tell() + read_buffer = fptr.read(8) + sub_length, sub_id = struct.unpack('>I4s', read_buffer) + sub_id = sub_id.decode('utf-8') + + # Regardless of whether or not we recognize the box, rewind back + # to properly advance to the next box. + fptr.seek(pos) + + # Now process any child boxes if we actually did recognize it. + if sub_id in _BOX_WITH_ID.keys(): + box.box = box.parse_superbox(fptr) + + return box + def parse_superbox(self, fptr): """Parse a superbox (box consisting of nothing but other boxes. @@ -159,36 +206,10 @@ class Jp2kBox(object): num_bytes, = struct.unpack('>Q', read_buffer) else: + # The box_length value really is the length of the box! num_bytes = box_length - # Call the proper parser for the given box with ID "T". - try: - box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError as err: - msg = 'Unrecognized box ({0}) encountered.'.format(box_id) - warnings.warn(msg) - box = Jp2kBox(box_id, offset=start, length=num_bytes, - longname='Unknown box') - - if fptr.tell() != start + 8: - # If the file pointer has advanced, then the KeyError - # ocurred during the parsing of the box. - pass - else: - # Could it be a superbox with recognizable child boxes? - # Peek ahead to see. - pos = fptr.tell() - read_buffer = fptr.read(8) - sub_length, sub_id = struct.unpack('>I4s', read_buffer) - sub_id = sub_id.decode('utf-8') - - # Regardless of whether or not we recognize the box, rewind back - # to properly advance to the next box. - fptr.seek(pos) - - # Now process any child boxes if we actually did recognize it. - if sub_id in _BOX_WITH_ID.keys(): - box.box = box.parse_superbox(fptr) + box = self.parse_this_box(fptr, box_id, start, num_bytes) superbox.append(box) From df89b5523aafd43cb0aead80c66ddb719c97da01 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 16 Feb 2014 22:16:46 -0500 Subject: [PATCH 56/83] Some pylint work. --- glymur/jp2box.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index d2e374b..ab8a6e5 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -63,6 +63,8 @@ class Jp2kBox(object): offset of the box from the start of the file. longname : str more verbose description of the box. + box : list + List of JPEG 2000 boxes. """ def __init__(self, box_id='', offset=0, length=0, longname=''): @@ -70,6 +72,7 @@ class Jp2kBox(object): self.length = length self.offset = offset self.longname = longname + self.box = [] def __repr__(self): msg = "glymur.jp2box.Jp2kBox(box_id='{0}', offset={1}, length={2}, " @@ -130,7 +133,7 @@ class Jp2kBox(object): box_id : str 4-letter identifier for the current box. start, num_bytes: int - Byte offset and length of the current box. + Byte offset and length of the current box. Returns ------- @@ -139,7 +142,7 @@ class Jp2kBox(object): """ try: box = _BOX_WITH_ID[box_id].parse(fptr, start, num_bytes) - except KeyError as err: + except KeyError: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) box = Jp2kBox(box_id, offset=start, length=num_bytes, @@ -154,7 +157,7 @@ class Jp2kBox(object): # Peek ahead to see. pos = fptr.tell() read_buffer = fptr.read(8) - sub_length, sub_id = struct.unpack('>I4s', read_buffer) + _, sub_id = struct.unpack('>I4s', read_buffer) sub_id = sub_id.decode('utf-8') # Regardless of whether or not we recognize the box, rewind back @@ -869,7 +872,7 @@ class ContiguousCodestreamBox(Jp2kBox): # Add indentation. strs = [('\n ' + x) for x in segstr.split('\n')] msg += ''.join(strs) - + return msg @staticmethod @@ -1725,8 +1728,7 @@ class PaletteBox(Jp2kBox): palette[j] = struct.unpack_from(fmt, read_buffer, offset=j * row_nbytes) - box = PaletteBox(palette, bps, signed, length=length, offset=offset) - return box + return PaletteBox(palette, bps, signed, length=length, offset=offset) # Map rreq codes to display text. From 5e76d1cf3927941b7563c3a52a006761447d6947 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 17 Feb 2014 17:38:26 -0500 Subject: [PATCH 57/83] Added print support for unknown boxes. #175 --- glymur/jp2box.py | 35 ++++++++++++++++++++++++-- glymur/test/fixtures.py | 8 ++++++ glymur/test/test_printing.py | 48 ++++++++++++++++++++---------------- 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index ab8a6e5..043a1be 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -145,8 +145,8 @@ class Jp2kBox(object): except KeyError: msg = 'Unrecognized box ({0}) encountered.'.format(box_id) warnings.warn(msg) - box = Jp2kBox(box_id, offset=start, length=num_bytes, - longname='Unknown box') + box = UnknownBox(box_id, offset=start, length=num_bytes, + longname='Unknown') if fptr.tell() != start + 8: # If the file pointer has advanced, then the KeyError @@ -2739,6 +2739,37 @@ class DataEntryURLBox(Jp2kBox): return box +class UnknownBox(Jp2kBox): + """Container for unrecognized boxes. + + Attributes + ---------- + box_id : str + 4-character identifier for the box. + length : int + length of the box in bytes. + offset : int + offset of the box from the start of the file. + longname : str + more verbose description of the box. + """ + def __init__(self, box_id, length=0, offset=-1, longname=''): + Jp2kBox.__init__(self, box_id=box_id, longname=longname) + self.length = length + self.offset = offset + + def __repr__(self): + msg = "glymur.jp2box.UnknownBox({0})".format(self.box_id) + return msg + + def __str__(self): + if len(self.box) > 0: + msg = self._str_superbox() + else: + msg = Jp2kBox.__str__(self) + return msg + + class UUIDBox(Jp2kBox): """Container for UUID box information. diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 6e0a76c..a7906c5 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -690,3 +690,11 @@ file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) Feature 060: 0x60 e-sRGB enumerated colorspace Feature 043: 0x40 Deprecated - compositing layer uses restricted ICC profile Vendor Features:""" + +file1_xml = r"""XML Box (xml ) @ (36, 439) + + + 2001-11-01T13:45:00.000-06:00 + Professional 120 Image + + """ diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 5fe3322..132d1ec 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -39,6 +39,7 @@ from .fixtures import text_gbr_27, text_gbr_33, text_gbr_34 class TestPrinting(unittest.TestCase): """Tests for verifying how printing works.""" def setUp(self): + self.jpxfile = glymur.data.jpxfile() self.jp2file = glymur.data.nemo() self.j2kfile = glymur.data.goodstuff() @@ -48,6 +49,31 @@ class TestPrinting(unittest.TestCase): def tearDown(self): pass + @unittest.skipIf(sys.hexversion < 0x03000000, "Needs unittest in 3.x.") + def test_unknown_superbox(self): + """Verify that we can handle an unknown superbox.""" + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with open(self.jpxfile, 'rb') as ifile: + tfile.write(ifile.read()) + + # Add the header for an unknwon superbox. + write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) + tfile.write(write_buffer) + write_buffer = struct.pack('>I4sI', 12, 'free'.encode(), 0) + tfile.write(write_buffer) + tfile.flush() + + with self.assertWarns(UserWarning): + jpx = Jp2k(tfile.name) + glymur.set_printoptions(short=True) + with patch('sys.stdout', new=StringIO()) as fake_out: + print(jpx.box[-1]) + actual = fake_out.getvalue().strip() + lines = ['Unknown Box (grp ) @ (695609, 20)', + ' Free Box (free) @ (695617, 12)'] + expected = '\n'.join(lines) + self.assertEqual(actual, expected) + def test_printoptions_bad_argument(self): """Verify error when bad parameter to set_printoptions""" with self.assertRaises(TypeError): @@ -602,27 +628,7 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(j.box[2]) actual = fake_out.getvalue().strip() - - lines = ['XML Box (xml ) @ (36, 439)', - ' ', - - ' ', - ' ' - + '2001-11-01T13:45:00.000-06:00' - + '', - - ' ' - + 'Professional 120 Image' - + '', - - ' ', - ' '] - expected = '\n'.join(lines) - self.assertEqual(actual, expected) + self.assertEqual(actual, fixtures.file1_xml) @unittest.skipIf(sys.hexversion < 0x03000000, "Only trusting python3 for printing non-ascii chars") From cd606e1f9df52dfe8207dac6fb15b7a3f5c7ae80 Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 18 Feb 2014 20:01:28 -0500 Subject: [PATCH 58/83] Starting write support for ftbl and flst boxes. #175 --- glymur/jp2box.py | 50 +++++++++++++++++++++-- glymur/test/test_jp2box_jpx.py | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 043a1be..4ce47ff 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -148,7 +148,8 @@ class Jp2kBox(object): box = UnknownBox(box_id, offset=start, length=num_bytes, longname='Unknown') - if fptr.tell() != start + 8: + cpos = fptr.tell() + if not ((cpos == start + 8) or (cpos == start + 16)): # If the file pointer has advanced, then the KeyError # ocurred during the parsing of the box. pass @@ -1122,6 +1123,18 @@ class FragmentListBox(Jp2kBox): self.length = length self.offset = offset + def _validate(self): + """Validate internal correctness.""" + if (((len(self.fragment_offset) != len(self.fragment_length)) or + (len(self.fragment_length) != len(self.data_reference)))): + msg = "The lengths of the fragment offsets, fragment lengths, and " + msg += "data reference items must be the same." + raise IOError(msg) + if any([x <= 0 for x in self.fragment_offset]): + raise IOError("Fragment offsets must all be positive.") + if any([x <= 0 for x in self.fragment_length]): + raise IOError("Fragment lengths must all be positive.") + def __repr__(self): msg = "glymur.jp2box.FragmentListBox()" return msg @@ -1141,6 +1154,22 @@ class FragmentListBox(Jp2kBox): return msg + def write(self, fptr): + """Write a fragment list box to file. + """ + self._validate() + num_items = len(self.fragment_offset) + length = 8 + 2 + num_items * 14 + fptr.write(struct.pack('>I', length)) + fptr.write(self.box_id.encode()) + fptr.write(struct.pack('>H', num_items)) + for j in range(num_items): + write_buffer = struct.pack('>QIH', + self.fragment_offset[j], + self.fragment_length[j], + self.data_reference[j]) + fptr.write(write_buffer) + @staticmethod def parse(fptr, offset, length): """Parse JPX free box. @@ -1184,10 +1213,11 @@ class FragmentTableBox(Jp2kBox): longname : str more verbose description of the box. """ - def __init__(self, length=0, offset=-1): + def __init__(self, box=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='ftbl', longname='Fragment Table') self.length = length self.offset = offset + self.box = box if box is not None else [] def __repr__(self): msg = "glymur.jp2box.FragmentTableBox()" @@ -1212,7 +1242,7 @@ class FragmentTableBox(Jp2kBox): Returns ------- - FreeBox instance + FragmentTableBox instance """ box = FragmentTableBox(length=length, offset=offset) @@ -1222,6 +1252,20 @@ class FragmentTableBox(Jp2kBox): return box + def _validate(self): + """Self-validate the box before writing.""" + box_ids = [box.box_id for box in self.box] + if len(box_ids) != 1 or box_ids[0] != 'flst': + msg = "Fragment table boxes must have a single fragment list " + msg += "box as a child box." + raise IOError(msg) + + def write(self, fptr): + """Write a fragment table box to file. + """ + self._validate() + self._write_superbox(fptr) + class FreeBox(Jp2kBox): diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 6624291..99250c5 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -41,6 +41,34 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_ftbl(self): + """Write a fragment table box.""" + # Add a negative test where offset < 0 + # Add a negative test where length < 0 + # Add a negative test where ref > 0 but no data reference box. + # Add a negative test where more than one flst + # Add negative test where ftbl contained in a superbox. + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + # The ftyp box must be modified to jpx. + boxes[1].brand = 'jpx ' + boxes[1].compatibility_list = ['jp2 ', 'jpxb'] + + offset = [89] + length = [1132288] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + ftbl = glymur.jp2box.FragmentTableBox(box=[flst]) + boxes.append(ftbl) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + jpx = jp2.wrap(tfile.name, boxes=boxes) + + self.assertEqual(jpx.box[1].compatibility_list, ['jp2 ', 'jpxb']) + self.assertEqual(jpx.box[-1].box_id, 'ftbl') + self.assertEqual(jpx.box[-1].box[0].box_id, 'flst') + def test_jpxb_compatibility(self): """Wrap JP2 to JPX, state jpxb compatibility""" jp2 = Jp2k(self.jp2file) @@ -194,6 +222,51 @@ class TestJPX(unittest.TestCase): def tearDown(self): pass + def test_flst_lens_not_the_same(self): + """A fragment list box items must be the same length.""" + offset = [89] + length = [1132288] + reference = [0, 0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_flst_offsets_not_positive(self): + """A fragment list box offsets must be positive.""" + offset = [0] + length = [1132288] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_flst_lengths_not_positive(self): + """A fragment list box lengths must be positive.""" + offset = [89] + length = [0] + reference = [0] + flst = glymur.jp2box.FragmentListBox(offset, length, reference) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + flst.write(tfile) + + def test_ftbl_boxes_empty(self): + """A fragment table box must have at least one child box.""" + ftbl = glymur.jp2box.FragmentTableBox() + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftbl.write(tfile) + + def test_ftbl_child_not_flst(self): + """A fragment table box can only contain a fragment list.""" + free = glymur.jp2box.FreeBox() + ftbl = glymur.jp2box.FragmentTableBox(box=[free]) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftbl.write(tfile) + def test_jpx_rreq_mask_length_3(self): """There are some JPX files with rreq mask length of 3.""" jpx = Jp2k(self.jpxfile) From c82d525dd2286d95811a078b3237cf58641563ab Mon Sep 17 00:00:00 2001 From: jevans Date: Wed, 19 Feb 2014 20:25:08 -0500 Subject: [PATCH 59/83] FileType box sanity checks. #175 --- glymur/jp2box.py | 13 +++++++++++++ glymur/test/test_jp2box.py | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 4ce47ff..0a370f9 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -1046,9 +1046,22 @@ class FileTypeBox(Jp2kBox): return msg + def _validate(self): + """Validate the box before writing to file.""" + if self.brand not in ['jp2 ', 'jpx ']: + msg = "The file type brand must be either 'jp2 ' or 'jpx '." + raise IOError(msg) + valid_cls = ['jp2 ', 'jpx ', 'jpxb'] + for item in self.compatibility_list: + if item not in valid_cls: + msg = "The file type compatibility list item '{0}' is not " + msg += "valid: valid entries are {1}" + raise IOError(msg.format(item, valid_cls)) + def write(self, fptr): """Write a File Type box to file. """ + self._validate() length = 16 + 4*len(self.compatibility_list) fptr.write(struct.pack('>I', length)) fptr.write('ftyp'.encode()) diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index feffec9..426bf66 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -340,6 +340,29 @@ class TestChannelDefinition(unittest.TestCase): association=association) +class TestFileTypeBox(unittest.TestCase): + """Test suite for ftyp box issues.""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_brand_unknown(self): + """A ftyp box brand must be 'jp2 ' or 'jpx '.""" + ftyp = glymur.jp2box.FileTypeBox(brand='jp3') + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftyp.write(tfile) + + def test_cl_entry_unknown(self): + """A ftyp box cl list can only contain 'jp2 ', 'jpx ', or 'jpxb'.""" + ftyp = glymur.jp2box.FileTypeBox(compatibility_list=['jp3']) + with self.assertRaises(IOError): + with tempfile.TemporaryFile() as tfile: + ftyp.write(tfile) + class TestColourSpecificationBox(unittest.TestCase): """Test suite for colr box instantiation.""" From 4276a978eab3e12516d9f077159b24b89048c002 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 20 Feb 2014 09:19:41 -0500 Subject: [PATCH 60/83] Allowing DataReferenceBoxes to be empty. #175 --- glymur/jp2box.py | 7 +++++-- glymur/test/test_jp2box_jpx.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 0a370f9..49cde5d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -915,9 +915,12 @@ class DataReferenceBox(Jp2kBox): DR : list Data Entry URL boxes. """ - def __init__(self, data_entry_url_boxes, length=0, offset=-1): + def __init__(self, data_entry_url_boxes=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='dtbl', longname='Data Reference') - self.DR = data_entry_url_boxes + if data_entry_url_boxes is None: + self.DR = [] + else: + self.DR = data_entry_url_boxes self.length = length self.offset = offset diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 99250c5..5d6cca2 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -127,6 +127,19 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box[2].box_id, 'lbl ') self.assertEqual(jpx.box[-1].box[2].label, label) + def test_empty_data_reference(self): + """Data reference boxes can be empty.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + dref = glymur.jp2box.DataReferenceBox() + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + jpx = jp2.wrap(tfile.name, boxes=boxes) + self.assertEqual(jpx.box[-1].box_id, 'dtbl') + self.assertEqual(len(jpx.box[-1].box), 0) + def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) From 4caa13a9559f4020f52fec56148803497acb9798 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 20 Feb 2014 22:34:52 -0500 Subject: [PATCH 61/83] More negative tests. #175 --- glymur/jp2box.py | 50 ++++++++++++++++++++++++++-------- glymur/test/test_jp2box.py | 12 +++++++- glymur/test/test_jp2box_jpx.py | 19 +++++++++++++ 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 49cde5d..f3fce20 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -29,6 +29,7 @@ import numpy as np from .codestream import Codestream from .core import _COLORSPACE_MAP_DISPLAY from .core import _COLOR_TYPE_MAP_DISPLAY +from .core import SRGB, GREYSCALE, YCC from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .core import ANY_ICC_PROFILE, VENDOR_COLOR_METHOD from .core import _pretty_print_xml @@ -270,13 +271,6 @@ class ColourSpecificationBox(Jp2kBox): approximation=0, colorspace=None, icc_profile=None, length=0, offset=-1): Jp2kBox.__init__(self, box_id='colr', longname='Colour Specification') - - if colorspace is not None and icc_profile is not None: - raise IOError("colorspace and icc_profile cannot both be set.") - if method not in (1, 2, 3, 4): - raise IOError("Invalid method.") - if approximation not in (0, 1, 2, 3, 4): - raise IOError("Invalid approximation.") self.method = method self.precedence = precedence self.approximation = approximation @@ -284,6 +278,33 @@ class ColourSpecificationBox(Jp2kBox): self.icc_profile = icc_profile self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + if self.colorspace is not None and self.icc_profile is not None: + raise IOError("colorspace and icc_profile cannot both be set.") + if self.method not in (1, 2, 3, 4): + raise IOError("Invalid method.") + if self.approximation not in (0, 1, 2, 3, 4): + raise IOError("Invalid approximation.") + + def _write_validate(self): + """In addition to constructor validation steps, run validation steps + for writing.""" + if self.colorspace is None: + msg = "Writing Colour Specification boxes without enumerated " + msg += "colorspaces is not supported at this time." + raise IOError(msg) + + if self.icc_profile is None: + if self.colorspace not in [SRGB, GREYSCALE, YCC]: + msg = "Colorspace should correspond to one of SRGB, GREYSCALE, " + msg += "or YCC." + raise IOError(msg) + + self._validate() + def __repr__(self): msg = "glymur.jp2box.ColourSpecificationBox(" @@ -327,10 +348,7 @@ class ColourSpecificationBox(Jp2kBox): def write(self, fptr): """Write an Colour Specification box to file. """ - if self.colorspace is None: - msg = "Writing Colour Specification boxes without enumerated " - msg += "colorspaces is not supported at this time." - raise NotImplementedError(msg) + self._write_validate() length = 15 if self.icc_profile is None else 11 + len(self.icc_profile) fptr.write(struct.pack('>I', length)) fptr.write('colr'.encode()) @@ -923,10 +941,20 @@ class DataReferenceBox(Jp2kBox): self.DR = data_entry_url_boxes self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + for box in self.DR: + if box.box_id != 'url ': + msg = 'All child boxes of a data reference box must be data ' + msg += 'entry URL boxes.' + raise IOError(msg) def write(self, fptr): """Write a Data Reference box to file. """ + self._validate() # Very similar to the say a superbox is written. orig_pos = fptr.tell() diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 426bf66..b6f0524 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -394,7 +394,7 @@ class TestColourSpecificationBox(unittest.TestCase): boxes = [self.jp2b, self.ftyp, self.jp2h, self.jp2c] boxes[2].box = [self.ihdr, ColourSpecificationBox(colorspace=None)] with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: - with self.assertRaises(NotImplementedError): + with self.assertRaises(IOError): j2k.wrap(tfile.name, boxes=boxes) @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") @@ -440,6 +440,16 @@ class TestColourSpecificationBox(unittest.TestCase): glymur.jp2box.ColourSpecificationBox(colorspace=colorspace, approximation=approx) + def test_colr_with_bad_color(self): + """colr must have a valid color, strange as though that may sound.""" + colorspace = -1 + approx = 0 + colr = glymur.jp2box.ColourSpecificationBox(colorspace=colorspace, + approximation=approx) + with tempfile.TemporaryFile() as tfile: + with self.assertRaises(IOError): + colr.write(tfile) + class TestAppend(unittest.TestCase): """Tests for append method.""" diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 5d6cca2..b2916c5 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -140,6 +140,25 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box_id, 'dtbl') self.assertEqual(len(jpx.box[-1].box), 0) + def test_deurl_child_of_dtbl(self): + """Data reference boxes can only contain data entry url boxes.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + ftyp = glymur.jp2box.FileTypeBox() + with self.assertRaises(IOError): + dref = glymur.jp2box.DataReferenceBox([ftyp]) + + # Try to get around it by appending the ftyp box after creation. + dref = glymur.jp2box.DataReferenceBox() + dref.DR.append(ftyp) + + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jpx = jp2.wrap(tfile.name, boxes=boxes) + def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) From f8ced317dbf3da38861e69ede3e6fe86f5d35604 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 22 Feb 2014 21:30:52 -0500 Subject: [PATCH 62/83] More negative tests. Incompatible change to ChannelDefinitionBox. #175 --- CHANGES.txt | 21 +++++++++-------- docs/source/changelog.rst | 1 + docs/source/how_do_i.rst | 2 +- glymur/jp2box.py | 47 +++++++++++++++++++++++--------------- glymur/test/test_jp2box.py | 32 +++++++++++++++++++++++++- 5 files changed, 73 insertions(+), 30 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1febdfb..24d9616 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,13 +1,14 @@ -Feb 09, 2014 - Removed support for Python 2.6. Added write support for JP2 - UUID, DataEntryURL, Palette and Component Mapping boxes, JPX - Association, NumberList and DataReference boxes. Added read - support for JPX free, number list, data reference, fragment - table, and fragment list boxes. Improved JPX Reader Requirements box - support. Added get_printoptions, set_printoptions functions. - Palette box now a 2D numpy array instead of a list of 1D arrays. - JP2 super box constructors now take optional box list argument. - Fixed bug where JPX files with more than one codestream but - advertising jp2 compatibility were not being read. +Feb 09, 2014 - Changed constructor for ChannelDefinition box. Removed support + for Python 2.6. Added write support for JP2 UUID, DataEntryURL, + Palette and Component Mapping boxes, JPX Association, NumberList + and DataReference boxes. Added read support for JPX free, + number list, data reference, fragment table, and fragment list + boxes. Improved JPX Reader Requirements box support. Added + get_printoptions, set_printoptions functions. Palette box now + a 2D numpy array instead of a list of 1D arrays. JP2 super box + constructors now take optional box list argument. Fixed bug + where JPX files with more than one codestream but advertising + jp2 compatibility were not being read. Jan 28, 2014 - v0.5.10 Fixed bad warning when reader requirements box mask length is unsupported. diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index f7e5921..ad4e40b 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -11,6 +11,7 @@ ChangeLog * added write support for JP2 UUID, dataEntryURL, palette, and component mapping boxes * added read/write support for JPX free, number list, and data reference boxes * Added read support for JPX fragment list and fragment table boxes + * incompatible change to channel definition box constructor, channel_type and association are no longer keyword arguments * incompatible change to palette box constructor, it now takes a 2D numpy array instead of a list of 1D arrays 0.5.0 (September 16, 2013) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index a1fe089..fa8f098 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -359,7 +359,7 @@ channel, but we aren't doing that). :: >>> from glymur.core import RED, GREEN, BLUE, WHOLE_IMAGE >>> asoc = [RED, GREEN, BLUE, WHOLE_IMAGE] - >>> cdef = glymur.jp2box.ChannelDefinitionBox(channel_type=ctype, association=asoc) + >>> cdef = glymur.jp2box.ChannelDefinitionBox(ctype, asoc) >>> print(cdef) Channel Definition Box (cdef) @ (0, 0) Channel 0 (color) ==> (1) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index f3fce20..f82151a 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -544,23 +544,28 @@ class ChannelDefinitionBox(Jp2kBox): association : list index of the associated color """ - def __init__(self, index=None, channel_type=None, association=None, - **kwargs): + def __init__(self, channel_type, association, index=None, **kwargs): Jp2kBox.__init__(self, box_id='cdef', longname='Channel Definition') - # channel type and association must be specified. - if channel_type is None or association is None: - raise IOError("channel_type and association must be specified.") - if index is None: - index = list(range(len(channel_type))) + self.index = tuple(range(len(channel_type))) + else: + self.index = tuple(index) - if len(index) != len(channel_type) or len(index) != len(association): + self.channel_type = tuple(channel_type) + self.association = tuple(association) + self.__dict__.update(**kwargs) + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + # channel type and association must be specified. + if not (len(self.index) == len(self.channel_type) == len(self.association)): msg = "Length of channel definition box inputs must be the same." raise IOError(msg) # channel types must be one of 0, 1, 2, 65535 - if any(x not in [0, 1, 2, 65535] for x in channel_type): + if any(x not in [0, 1, 2, 65535] for x in self.channel_type): msg = "Channel types must be in the set of\n\n" msg += " 0 - colour image data for associated color\n" msg += " 1 - opacity\n" @@ -568,10 +573,6 @@ class ChannelDefinitionBox(Jp2kBox): msg += " 65535 - unspecified" raise IOError(msg) - self.index = tuple(index) - self.channel_type = tuple(channel_type) - self.association = tuple(association) - self.__dict__.update(**kwargs) def __str__(self): msg = Jp2kBox.__str__(self) @@ -597,6 +598,7 @@ class ChannelDefinitionBox(Jp2kBox): def write(self, fptr): """Write a channel definition box to file. """ + self._validate() num_components = len(self.association) fptr.write(struct.pack('>I', 8 + 2 + num_components * 6)) fptr.write('cdef'.encode('utf-8')) @@ -634,9 +636,10 @@ class ChannelDefinitionBox(Jp2kBox): channel_type = data[1:num_components * 6:3] association = data[2:num_components * 6:3] - box = ChannelDefinitionBox(index=index, channel_type=channel_type, - association=association, length=length, - offset=offset) + box = ChannelDefinitionBox(index=tuple(index), + channel_type=tuple(channel_type), + association=tuple(association), + length=length, offset=offset) return box @@ -795,7 +798,6 @@ class ComponentMappingBox(Jp2kBox): if _printoptions['short'] == True: return msg - for k in range(len(self.component_index)): if self.mapping_type[k] == 1: msg += '\n Component {0} ==> palette column {1}' @@ -1050,7 +1052,6 @@ class FileTypeBox(Jp2kBox): self.brand = brand self.minor_version = minor_version if compatibility_list is None: - # see W0102, pylint self.compatibility_list = ['jp2 '] else: self.compatibility_list = compatibility_list @@ -1718,6 +1719,15 @@ class PaletteBox(Jp2kBox): self.signed = signed self.length = length self.offset = offset + self._validate() + + def _validate(self): + """Verify that the box obeys the specifications.""" + if ((len(self.bits_per_component) != len(self.signed)) or + (len(self.signed) != self.palette.shape[1])): + msg = "The length of the 'bits_per_component' and the 'signed' " + msg += "members must equal the number of columns of the palette." + raise IOError(msg) def __repr__(self): msg = "glymur.jp2box.PaletteBox({0}, bits_per_component={1}, " @@ -1737,6 +1747,7 @@ class PaletteBox(Jp2kBox): def write(self, fptr): """Write a Palette box to file. """ + self._validate() bytes_per_row = sum(self.bits_per_component) / 8 bytes_per_palette = bytes_per_row * self.palette.shape[0] box_length = 8 + 3 + self.palette.shape[1] + bytes_per_palette diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index b6f0524..fb15e26 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -139,7 +139,7 @@ class TestChannelDefinition(unittest.TestCase): def test_cdef_no_inputs(self): """channel_type and association are required inputs.""" - with self.assertRaises(IOError): + with self.assertRaises(TypeError): glymur.jp2box.ChannelDefinitionBox() def test_rgb_with_index(self): @@ -451,6 +451,36 @@ class TestColourSpecificationBox(unittest.TestCase): colr.write(tfile) +@unittest.skipIf(os.name == "nt", + "Problems using NamedTemporaryFile on windows.") +class TestPaletteBox(unittest.TestCase): + """Test suite for pclr box instantiation.""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def test_mismatched_bitdepth_signed(self): + """bitdepth and signed arguments must have equal length""" + palette = np.array([[255, 0, 255], [0, 255, 0]], dtype=np.uint8) + bps = (8, 8, 8) + signed = (False, False) + with self.assertRaises(IOError): + pclr = glymur.jp2box.PaletteBox(palette, bits_per_component=bps, + signed=signed) + + def test_mismatched_signed_palette(self): + """bitdepth and signed arguments must have equal length""" + palette = np.array([[255, 0, 255], [0, 255, 0]], dtype=np.uint8) + bps = (8, 8, 8, 8) + signed = (False, False, False, False) + with self.assertRaises(IOError): + pclr = glymur.jp2box.PaletteBox(palette, bits_per_component=bps, + signed=signed) + + class TestAppend(unittest.TestCase): """Tests for append method.""" From 16ce2ef88380dcad85e2edfda957b46ec9a29948 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 23 Feb 2014 12:37:47 -0500 Subject: [PATCH 63/83] More negative tests, pylint work. #175 --- glymur/jp2box.py | 17 ++++++--- glymur/jp2k.py | 17 ++++++++- glymur/test/test_jp2box_jpx.py | 65 ++++++++++++++++++++++------------ 3 files changed, 72 insertions(+), 27 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index f82151a..18f994d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -124,7 +124,7 @@ class Jp2kBox(object): fptr.write(struct.pack('>I', end_pos - orig_pos)) fptr.seek(end_pos) - def parse_this_box(self, fptr, box_id, start, num_bytes): + def _parse_this_box(self, fptr, box_id, start, num_bytes): """Parse the current box. Parameters @@ -214,7 +214,7 @@ class Jp2kBox(object): # The box_length value really is the length of the box! num_bytes = box_length - box = self.parse_this_box(fptr, box_id, start, num_bytes) + box = self._parse_this_box(fptr, box_id, start, num_bytes) superbox.append(box) @@ -560,7 +560,8 @@ class ChannelDefinitionBox(Jp2kBox): def _validate(self): """Verify that the box obeys the specifications.""" # channel type and association must be specified. - if not (len(self.index) == len(self.channel_type) == len(self.association)): + if not ((len(self.index) == len(self.channel_type)) and + (len(self.channel_type) == len(self.association))): msg = "Length of channel definition box inputs must be the same." raise IOError(msg) @@ -953,10 +954,18 @@ class DataReferenceBox(Jp2kBox): msg += 'entry URL boxes.' raise IOError(msg) + def _write_validate(self): + """Verify that the box obeys the specifications for writing. + """ + if len(self.DR) == 0: + msg = "A data reference box cannot be empty when written to a file." + raise IOError(msg) + self._validate() + def write(self, fptr): """Write a Data Reference box to file. """ - self._validate() + self._write_validate() # Very similar to the say a superbox is written. orig_pos = fptr.tell() diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 96b6e68..988e814 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1110,6 +1110,10 @@ def _validate_nonzero_image_size(nrows, ncols, component_index): raise IOError(msg) +JP2_IDS = ['colr', 'cdef', 'cmap', 'jp2c', 'ftyp', 'ihdr', 'jp2h', 'jP ', + 'pclr', 'res ', 'resc', 'resd', 'xml ', 'ulst', 'uinf', 'url ', + 'uuid'] + def _validate_jp2_box_sequence(boxes): """Run through series of tests for JP2 box legality. @@ -1118,6 +1122,18 @@ def _validate_jp2_box_sequence(boxes): _validate_signature_compatibility(boxes) _validate_jp2h(boxes) _validate_jp2c(boxes) + if boxes[1].brand == 'jpx ': + _validate_jpx_box_sequence(boxes) + else: + count = _collect_box_count(boxes) + for id in count.keys(): + if id not in JP2_IDS: + msg = "The presence of a '{0}' box requires that the file type " + msg += "brand be set to 'jpx '." + raise IOError(msg.format(id)) + +def _validate_jpx_box_sequence(boxes): + """Run through series of tests for JPX box legality.""" _validate_association(boxes) _validate_label(boxes) _validate_jpx_brand(boxes, boxes[1].brand) @@ -1125,7 +1141,6 @@ def _validate_jp2_box_sequence(boxes): _validate_singletons(boxes) _validate_top_level(boxes) - def _validate_signature_compatibility(boxes): """Validate the file signature and compatibility status.""" # Check for a bad sequence of boxes. diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index b2916c5..e50d4f9 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -4,12 +4,10 @@ Test suite specifically targeting JPX box layout. """ import os -import shutil import struct import sys import tempfile import unittest -import warnings import xml.etree.cElementTree as ET import glymur @@ -41,6 +39,18 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_jp2_with_jpx_box(self): + """If the brand is jp2, then no jpx boxes are allowed.""" + jp2 = Jp2k(self.jp2file) + boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + boxes = jp2.box + + boxes.append(glymur.jp2box.AssociationBox()) + + with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: + with self.assertRaises(IOError): + jp2.wrap(tfile.name, boxes=boxes) + def test_ftbl(self): """Write a fragment table box.""" # Add a negative test where offset < 0 @@ -128,17 +138,18 @@ class TestJPXWrap(unittest.TestCase): self.assertEqual(jpx.box[-1].box[2].label, label) def test_empty_data_reference(self): - """Data reference boxes can be empty.""" + """Empty data reference boxes can be created, but not written.""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + boxes[1].brand = 'jpx ' + dref = glymur.jp2box.DataReferenceBox() boxes.append(dref) with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: - jpx = jp2.wrap(tfile.name, boxes=boxes) - self.assertEqual(jpx.box[-1].box_id, 'dtbl') - self.assertEqual(len(jpx.box[-1].box), 0) + with self.assertRaises(IOError): + jp2.wrap(tfile.name, boxes=boxes) def test_deurl_child_of_dtbl(self): """Data reference boxes can only contain data entry url boxes.""" @@ -157,13 +168,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_only_one_data_reference(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + flag = 0 version = (0, 0, 0) url = 'file:////usr/local/bin' @@ -174,16 +188,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_lbl_at_top_level(self): """Label boxes can only be inside a asoc box .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] - flag = 0 - version = (0, 0, 0) - url = 'file:////usr/local/bin' + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + lblb = glymur.jp2box.LabelBox('hi there') # Put it inside the jp2 header box. @@ -191,13 +205,16 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) - def test_data_reference_not_at_top_level(self): + def test_data_reference_in_subbox(self): """Data reference boxes cannot be inside a superbox .""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' + flag = 0 version = (0, 0, 0) url = 'file:////usr/local/bin' @@ -209,13 +226,17 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(IOError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_jp2_to_jpx_sans_jp2_compatibility(self): """jp2 wrapped to jpx not including jp2 compatibility is wrong.""" jp2 = Jp2k(self.jp2file) boxes = [jp2.box[idx] for idx in [0, 1, 2, 4]] + + # Have to make the ftyp brand jpx. + boxes[1].brand = 'jpx ' boxes[1].compatibility_list.append('jp2 ') + numbers = [0, 1] nlst = glymur.jp2box.NumberListBox(numbers) the_xml = ET.fromstring('0') @@ -225,7 +246,7 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(RuntimeError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) def test_jp2_to_jpx_sans_jpx_brand(self): """Verify error when jp2 wrapped to jpx does not include jpx brand.""" @@ -241,7 +262,7 @@ class TestJPXWrap(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix=".jpx") as tfile: with self.assertRaises(RuntimeError): - jpx = jp2.wrap(tfile.name, boxes=boxes) + jp2.wrap(tfile.name, boxes=boxes) @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") @@ -262,7 +283,7 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_flst_offsets_not_positive(self): """A fragment list box offsets must be positive.""" @@ -272,7 +293,7 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_flst_lengths_not_positive(self): """A fragment list box lengths must be positive.""" @@ -282,14 +303,14 @@ class TestJPX(unittest.TestCase): flst = glymur.jp2box.FragmentListBox(offset, length, reference) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - flst.write(tfile) + flst.write(tfile) def test_ftbl_boxes_empty(self): """A fragment table box must have at least one child box.""" ftbl = glymur.jp2box.FragmentTableBox() with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - ftbl.write(tfile) + ftbl.write(tfile) def test_ftbl_child_not_flst(self): """A fragment table box can only contain a fragment list.""" @@ -297,7 +318,7 @@ class TestJPX(unittest.TestCase): ftbl = glymur.jp2box.FragmentTableBox(box=[free]) with self.assertRaises(IOError): with tempfile.TemporaryFile() as tfile: - ftbl.write(tfile) + ftbl.write(tfile) def test_jpx_rreq_mask_length_3(self): """There are some JPX files with rreq mask length of 3.""" @@ -314,7 +335,7 @@ class TestJPX(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: with open(self.jpxfile, 'rb') as ifile: tfile.write(ifile.read()) - + # Add the header for an unknwon superbox. write_buffer = struct.pack('>I4s', 20, 'grp '.encode()) tfile.write(write_buffer) From 992c036da701ab297bca4bada4d7e3507c9ed5a1 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 24 Feb 2014 21:12:24 -0500 Subject: [PATCH 64/83] Added ftbl/data reference test. Removed bad asoc check. #175 --- glymur/jp2k.py | 24 ++++++------------------ glymur/test/test_jp2box_jpx.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 988e814..6db0198 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -1134,7 +1134,6 @@ def _validate_jp2_box_sequence(boxes): def _validate_jpx_box_sequence(boxes): """Run through series of tests for JPX box legality.""" - _validate_association(boxes) _validate_label(boxes) _validate_jpx_brand(boxes, boxes[1].brand) _validate_jpx_compatibility(boxes, boxes[1].compatibility_list) @@ -1283,6 +1282,12 @@ def _validate_top_level(boxes): if 'dtbl' in multiples: raise IOError('There can only be one dtbl box in a file.') + # If there is one data reference box, then there must also be one ftbl. + if 'dtbl' in count and 'ftbl' not in count: + msg = 'The presence of a data reference box requires the presence of ' + msg += 'a fragment table box as well.' + raise IOError(msg) + def _validate_singletons(boxes): """Several boxes can only occur once.""" count = _collect_box_count(boxes) @@ -1336,23 +1341,6 @@ def _validate_label(boxes): # Same set of checks on any child boxes. _validate_label(box.box) -def _validate_association(boxes): - """ - Association boxes can only contain number list boxes and xml boxes, as far - as we know. - """ - for box in boxes: - if box.box_id == 'asoc': - if box.box[0].box_id != 'nlst' or box.box[1].box_id != 'xml ': - msg = "An Association box can only contain a NumberList box " - msg += "followed by an XML box." - raise RuntimeError(msg) - if hasattr(box, 'box') != 0: - # Same set of checks on any child boxes. - _validate_association(box.box) - - - def extract_image_cube(image): """Extract 3D image from openjpeg data structure. """ diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index e50d4f9..1eb5198 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -270,6 +270,7 @@ class TestJPX(unittest.TestCase): """Test suite for other JPX boxes.""" def setUp(self): + self.jp2file = glymur.data.nemo() self.jpxfile = glymur.data.jpxfile() def tearDown(self): @@ -354,6 +355,25 @@ class TestJPX(unittest.TestCase): self.assertEqual(j.box[16].box[0].box_id, 'free') self.assertEqual(type(j.box[16].box[0]), glymur.jp2box.FreeBox) + def test_data_reference_requires_dtbl(self): + """The existance of a data reference box requires a ftbl box as well.""" + flag = 0 + version = (0, 0, 0) + url1 = 'file:////usr/local/bin' + url2 = 'http://glymur.readthedocs.org' + jpx1 = glymur.Jp2k(self.jp2file) + boxes = jpx1.box + boxes[1].brand = 'jpx ' + + deurl1 = glymur.jp2box.DataEntryURLBox(flag, version, url1) + deurl2 = glymur.jp2box.DataEntryURLBox(flag, version, url2) + dref = glymur.jp2box.DataReferenceBox([deurl1, deurl2]) + boxes.append(dref) + + with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: + with self.assertRaises(IOError): + jpx2 = jpx1.wrap(tfile.name, boxes=boxes) + def test_dtbl(self): """Verify that we can interpret Data Reference boxes.""" # Copy the existing JPX file, add a data reference box onto the end. From 99d1a584d8619f75c7218ba920f935886c44eb8a Mon Sep 17 00:00:00 2001 From: John Evans Date: Tue, 25 Feb 2014 07:15:12 -0500 Subject: [PATCH 65/83] Verifying that data entry url box URLs are null terminated. #175 --- glymur/jp2box.py | 9 +++++++-- glymur/test/test_jp2box.py | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 18f994d..3cf637d 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -2788,13 +2788,18 @@ class DataEntryURLBox(Jp2kBox): def write(self, fptr): """Write a data entry url box to file. """ - length = 8 + 1 + 3 + len(self.url.encode()) + # Make sure it is written out as null-terminated. + url = self.url + if self.url[-1] != chr(0): + url = url + chr(0) + + length = 8 + 1 + 3 + len(url.encode()) write_buffer = struct.pack('>I4sBBBB', length, self.box_id.encode(), self.version, self.flag[0], self.flag[1], self.flag[2]) fptr.write(write_buffer) - fptr.write(self.url.encode()) + fptr.write(url.encode()) def __repr__(self): diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index fb15e26..5861971 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -77,6 +77,28 @@ class TestDataEntryURL(unittest.TestCase): self.assertEqual(jp22.box[4].flag, (0, 0, 0)) self.assertEqual(jp22.box[4].url, url) + def test_null_termination(self): + """I.9.3.2 specifies that the location field must be null terminated.""" + jp2 = Jp2k(self.jp2file) + + url = 'http://glymur.readthedocs.org' + deurl = glymur.jp2box.DataEntryURLBox(0, (0, 0, 0), url) + boxes = [box for box in jp2.box if box.box_id != 'uuid'] + boxes.append(deurl) + with tempfile.NamedTemporaryFile(suffix=".jp2") as tfile: + jp22 = jp2.wrap(tfile.name, boxes=boxes) + + self.assertEqual(jp22.box[-1].length, 42) + + # Go to the last box. Seek past the L, T, version, and flag fields. + with open(tfile.name, 'rb') as fptr: + fptr.seek(jp22.box[-1].offset + 4 + 4 + 1 + 3) + + nbytes = jp22.box[-1].offset + jp22.box[-1].length - fptr.tell() + read_buffer = fptr.read(nbytes) + read_url = read_buffer.decode('utf-8') + self.assertEqual(url + chr(0), read_url) + @unittest.skipIf(glymur.version.openjpeg_version_tuple[0] < 2 or OPENJP2_IS_V2_OFFICIAL, From 04b143e1ff59cae24b9ad390d321d3e253535e4a Mon Sep 17 00:00:00 2001 From: jevans Date: Tue, 25 Feb 2014 21:41:02 -0500 Subject: [PATCH 66/83] Added proof of concept test for linking exterior codestreams. Closes #175 --- glymur/test/test_jp2box_jpx.py | 60 ++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 1eb5198..7493dea 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -12,7 +12,8 @@ import xml.etree.cElementTree as ET import glymur from glymur import Jp2k - +from glymur.jp2box import DataEntryURLBox, FileTypeBox, JPEG2000SignatureBox +from glymur.jp2box import DataReferenceBox, FragmentListBox, FragmentTableBox @unittest.skipIf(os.name == "nt", "Temporary file issue on window.") class TestJPXWrap(unittest.TestCase): @@ -20,6 +21,7 @@ class TestJPXWrap(unittest.TestCase): def setUp(self): self.jp2file = glymur.data.nemo() + self.j2kfile = glymur.data.goodstuff() raw_xml = b""" @@ -39,6 +41,60 @@ class TestJPXWrap(unittest.TestCase): def tearDown(self): os.unlink(self.xmlfile) + def test_jpx_ftbl_no_codestream(self): + """Can have a jpx with no codestream.""" + with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile1: + with open(self.jp2file, 'rb') as fptr: + tfile1.write(fptr.read()) + tfile1.flush() + jp2_1 = Jp2k(tfile1.name) + + jp2c = [box for box in jp2_1.box if box.box_id == 'jp2c'][0] + + # coff and clen will be the offset and length input arguments + # to the fragment list box. dr_idx is the data reference index. + coff = [] + clen = [] + dr_idx = [] + + coff.append(jp2c.offset + 8) + clen.append(jp2c.length - (coff[0] - jp2c.offset)) + dr_idx.append(1) + + # Make the url box for this codestream. + url1 = DataEntryURLBox(0, [0, 0, 0], 'file://' + tfile1.name) + + with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile2: + + j2k = Jp2k(self.j2kfile) + jp2_2 = j2k.wrap(tfile2.name) + + jp2c = [box for box in jp2_2.box if box.box_id == 'jp2c'][0] + coff.append(jp2c.offset + 8) + clen.append(jp2c.length - (coff[0] - jp2c.offset)) + dr_idx.append(2) + + # Make the url box for this codestream. + url2 = DataEntryURLBox(0, [0, 0, 0], 'file://' + tfile2.name) + + boxes = [JPEG2000SignatureBox(), + FileTypeBox(brand='jpx ', + compatibility_list=['jpx ', + 'jp2 ', 'jpxb']), + jp2_1.box[2]] + with tempfile.NamedTemporaryFile(suffix='.jpx') as tjpx: + for box in boxes: + box.write(tjpx) + + flst = FragmentListBox(coff, clen, dr_idx) + ftbl = FragmentTableBox([flst]) + ftbl.write(tjpx) + + boxes = [url1, url2] + dtbl = DataReferenceBox(data_entry_url_boxes=boxes) + dtbl.write(tjpx) + tjpx.flush() + def test_jp2_with_jpx_box(self): """If the brand is jp2, then no jpx boxes are allowed.""" jp2 = Jp2k(self.jp2file) @@ -372,7 +428,7 @@ class TestJPX(unittest.TestCase): with tempfile.NamedTemporaryFile(suffix='.jpx') as tfile: with self.assertRaises(IOError): - jpx2 = jpx1.wrap(tfile.name, boxes=boxes) + jpx1.wrap(tfile.name, boxes=boxes) def test_dtbl(self): """Verify that we can interpret Data Reference boxes.""" From 63fdde230db21774adfeebfe35d3390ef160b81b Mon Sep 17 00:00:00 2001 From: jevans Date: Wed, 26 Feb 2014 20:56:25 -0500 Subject: [PATCH 67/83] Migrated from ElementTree to lxml. #176 --- glymur/_uuid_io.py | 3 +- glymur/core.py | 8 +- glymur/jp2box.py | 13 +- glymur/test/fixtures.py | 524 ++++++++++++++++---------------- glymur/test/test_jp2box.py | 4 +- glymur/test/test_jp2box_jpx.py | 2 +- glymur/test/test_jp2box_uuid.py | 5 +- glymur/test/test_jp2box_xml.py | 3 +- glymur/test/test_printing.py | 15 +- 9 files changed, 296 insertions(+), 281 deletions(-) diff --git a/glymur/_uuid_io.py b/glymur/_uuid_io.py index ed610e5..399fa99 100644 --- a/glymur/_uuid_io.py +++ b/glymur/_uuid_io.py @@ -7,7 +7,8 @@ import re import struct import sys import warnings -import xml.etree.cElementTree as ET + +import lxml.etree as ET if sys.hexversion < 0x02070000: # pylint: disable=F0401,E0611 diff --git a/glymur/core.py b/glymur/core.py index 620d0e3..30282f2 100644 --- a/glymur/core.py +++ b/glymur/core.py @@ -1,7 +1,7 @@ """Core definitions to be shared amongst the modules. """ import copy -import xml.etree.cElementTree as ET +import lxml.etree as ET # Progression order LRCP = 0 @@ -81,9 +81,9 @@ _CAPABILITIES_DISPLAY = { def _pretty_print_xml(xml, level=0): """Pretty print XML data. """ - xml = copy.deepcopy(xml) - _indent(xml.getroot(), level=level) - xmltext = ET.tostring(xml.getroot(), encoding='utf-8').decode('utf-8') + #xml = copy.deepcopy(xml) + #_indent(xml.getroot(), level=level) + xmltext = ET.tostring(xml, encoding='utf-8').decode('utf-8') # Indent it a bit. lst = [(' ' + x) for x in xmltext.split('\n')] diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 3cf637d..192763f 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -22,8 +22,8 @@ import struct import sys import uuid import warnings -import xml.etree.cElementTree as ET +import lxml.etree as ET import numpy as np from .codestream import Codestream @@ -2565,7 +2565,10 @@ class XMLBox(Jp2kBox): xml = self.xml if self.xml is not None: - msg += _pretty_print_xml(self.xml) + bstr = ET.tostring(self.xml, + encoding='utf-8', + pretty_print=True).decode('utf-8') + msg += '\n {0}'.format(bstr) else: msg += '\n {0}'.format(xml) return msg @@ -2971,8 +2974,10 @@ class UUIDBox(Jp2kBox): return msg if self.uuid == uuid.UUID('be7acfcb-97a9-42e8-9c71-999491e3afac'): - line = '\n UUID Data: {0}' - xmlstring = _pretty_print_xml(self.data) + line = '\n UUID Data:\n{0}' + xmlstring = ET.tostring(self.data, + encoding='utf-8', + pretty_print=True).decode('utf-8') xmlstring = xmlstring.rstrip() msg += line.format(xmlstring) elif self.uuid.bytes == b'JpgTiffExif->JP2': diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index a7906c5..e7c81a3 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -183,90 +183,92 @@ def read_pgx_header(pgx_file): nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - """ + UUID Data: + + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + +""" SimpleRDF = """ - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - + UUID Data: + + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + + Contiguous Codestream Box (jp2c) @ (3223, 1132296) Main header: SOC marker segment @ (3231, 0) @@ -575,90 +579,92 @@ JP2 Header Box (jp2h) @ (32, 45) Colorspace: sRGB UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - + UUID Data: + + + + + Google + 2013-02-09T14:47:53 + + + 1 + 72/1 + 72/1 + 2 + HTC + HTC Glacier + 2592 + 1456 + + + 8 + 8 + 8 + + + 2 + 3 + + + 1343036288/4294967295 + 1413044224/4294967295 + + + + + 2748779008/4294967295 + 1417339264/4294967295 + 1288490240/4294967295 + 2576980480/4294967295 + 644245120/4294967295 + 257698032/4294967295 + + + + + 1 + 2528 + 1424 + 353/100 + 0 + 0/1 + WGS-84 + 2013-02-09T14:47:53 + + + 76 + + + 0220 + 0100 + + + 1 + 2 + 3 + 0 + + + 42,20.56N + 71,5.29W + 2013-02-09T19:47:53Z + NETWORK + + + 2013-02-09T14:47:53 + + + + + Glymur + Python XMP Toolkit + + + + + + Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" nemo_dump_no_codestream_no_xml = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) @@ -691,10 +697,10 @@ file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) Feature 043: 0x40 Deprecated - compositing layer uses restricted ICC profile Vendor Features:""" -file1_xml = r"""XML Box (xml ) @ (36, 439) - - - 2001-11-01T13:45:00.000-06:00 - Professional 120 Image - - """ +file1_xml = """XML Box (xml ) @ (36, 439) + +\t +\t\t2001-11-01T13:45:00.000-06:00 +\t\tProfessional 120 Image +\t +""" diff --git a/glymur/test/test_jp2box.py b/glymur/test/test_jp2box.py index 5861971..5dedbcc 100644 --- a/glymur/test/test_jp2box.py +++ b/glymur/test/test_jp2box.py @@ -22,10 +22,10 @@ import sys import tempfile import uuid from uuid import UUID -import xml.etree.cElementTree as ET import unittest import warnings +import lxml.etree as ET import numpy as np import glymur @@ -1056,7 +1056,7 @@ class TestRepr(unittest.TestCase): box = glymur.jp2box.XMLBox(xml=tree) regexp = r"""glymur.jp2box.XMLBox""" - regexp += r"""\(xml=<(xml.etree.ElementTree.){0,1}ElementTree object """ + regexp += r"""\(xml=\)""" if sys.hexversion < 0x03000000: diff --git a/glymur/test/test_jp2box_jpx.py b/glymur/test/test_jp2box_jpx.py index 7493dea..f8e9db8 100644 --- a/glymur/test/test_jp2box_jpx.py +++ b/glymur/test/test_jp2box_jpx.py @@ -8,7 +8,7 @@ import struct import sys import tempfile import unittest -import xml.etree.cElementTree as ET +import lxml.etree as ET import glymur from glymur import Jp2k diff --git a/glymur/test/test_jp2box_uuid.py b/glymur/test/test_jp2box_uuid.py index 82dc253..a904174 100644 --- a/glymur/test/test_jp2box_uuid.py +++ b/glymur/test/test_jp2box_uuid.py @@ -18,7 +18,6 @@ import sys import tempfile import uuid import warnings -import xml.etree if sys.hexversion < 0x02070000: import unittest2 as unittest @@ -35,6 +34,8 @@ if sys.hexversion <= 0x03030000: else: from unittest.mock import patch +import lxml.etree + from .fixtures import HAS_PYTHON_XMP_TOOLKIT, OPJ_DATA_ROOT if HAS_PYTHON_XMP_TOOLKIT: from libxmp import XMPMeta @@ -71,7 +72,7 @@ class TestUUIDXMP(unittest.TestCase): # The data should be an XMP packet, which gets interpreted as # an ElementTree. self.assertTrue(isinstance(jp2.box[-1].data, - xml.etree.ElementTree.ElementTree)) + lxml.etree._ElementTree)) class TestUUIDExif(unittest.TestCase): """Tests for UUIDs of Exif type.""" diff --git a/glymur/test/test_jp2box_xml.py b/glymur/test/test_jp2box_xml.py index 4950da4..facc4df 100644 --- a/glymur/test/test_jp2box_xml.py +++ b/glymur/test/test_jp2box_xml.py @@ -20,7 +20,6 @@ import sys import tempfile import unittest import warnings -import xml.etree.cElementTree as ET if sys.hexversion < 0x03000000: from StringIO import StringIO @@ -32,6 +31,8 @@ if sys.hexversion <= 0x03030000: else: from unittest.mock import patch +import lxml.etree as ET + import glymur from glymur import Jp2k from glymur.jp2box import ColourSpecificationBox, ContiguousCodestreamBox diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 132d1ec..45b21df 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -15,7 +15,6 @@ import struct import sys import tempfile import warnings -from xml.etree import cElementTree as ET import unittest if sys.hexversion < 0x03000000: @@ -28,10 +27,12 @@ if sys.hexversion <= 0x03030000: else: from unittest.mock import patch +import lxml.etree as ET + import glymur from glymur import Jp2k from . import fixtures -from .fixtures import OPJ_DATA_ROOT, opj_data_file, nemo_xmp_box +from .fixtures import OPJ_DATA_ROOT, opj_data_file from .fixtures import text_gbr_27, text_gbr_33, text_gbr_34 @@ -118,6 +119,7 @@ class TestPrinting(unittest.TestCase): lst = actual.split('\n') lst = lst[1:] actual = '\n'.join(lst) + self.maxDiff = None self.assertEqual(actual, fixtures.nemo_dump_no_codestream) def test_printoptions_no_xml(self): @@ -209,6 +211,7 @@ class TestPrinting(unittest.TestCase): lst = actual.split('\n') lst = lst[1:] actual = '\n'.join(lst) + self.maxDiff = None self.assertEqual(actual, fixtures.nemo_dump_full) def test_entire_file(self): @@ -567,7 +570,7 @@ class TestPrinting(unittest.TestCase): print(j.box[3]) actual = fake_out.getvalue().strip() - expected = nemo_xmp_box + expected = fixtures.nemo_xmp_box self.assertEqual(actual, expected) def test_codestream(self): @@ -639,8 +642,7 @@ class TestPrinting(unittest.TestCase): # # 2.7.5 (fedora 19) prints xml entities. # 2.7.3 seems to want to print hex escapes. - text = u""" - Strömung""" + text = u"""Strömung""" if sys.hexversion < 0x03000000: xml = ET.parse(StringIO(text.encode('utf-8'))) else: @@ -668,8 +670,7 @@ class TestPrinting(unittest.TestCase): # # 2.7.5 (fedora 19) prints xml entities. # 2.7.3 seems to want to print hex escapes. - text = u""" - Россия""" + text = u"""Россия""" if sys.hexversion < 0x03000000: xml = ET.parse(StringIO(text.encode('utf-8'))) else: From f176204b2cd914496133720ea7fb6cd2eeb5371b Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 27 Feb 2014 06:55:30 -0500 Subject: [PATCH 68/83] Updated requirements for lxml. #176 --- .travis.yml | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f45864..c055dce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,8 @@ before_install: # command to install dependencies install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --use-mirrors contextlib2 mock; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then pip install --use-mirrors numpy; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install --use-mirrors lxml contextlib2 mock; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then pip install --use-mirrors lxml numpy; fi # command to run tests script: diff --git a/setup.py b/setup.py index b1332d8..41deb1a 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ kwargs = {'name': 'Glymur', 'license': 'MIT', 'test_suite': 'glymur.test'} -instllrqrs = ['numpy>=1.4.1'] +instllrqrs = ['lxml>=3.3', 'numpy>=1.4.1'] if sys.hexversion < 0x03030000: instllrqrs.append('contextlib2>=0.4') instllrqrs.append('mock>=1.0.1') From e8a0681a99ac9e12e19049c7cf307dad55eb9d09 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 27 Feb 2014 11:07:02 -0500 Subject: [PATCH 69/83] Confirmed to work with lxml 3.2.3, down from 3.3. #176 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 41deb1a..7769392 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ kwargs = {'name': 'Glymur', 'license': 'MIT', 'test_suite': 'glymur.test'} -instllrqrs = ['lxml>=3.3', 'numpy>=1.4.1'] +instllrqrs = ['lxml>=3.2.3', 'numpy>=1.4.1'] if sys.hexversion < 0x03030000: instllrqrs.append('contextlib2>=0.4') instllrqrs.append('mock>=1.0.1') From 5943aa6c7a1ac63b366f965a541cac4ea5d391c5 Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 27 Feb 2014 18:48:48 -0500 Subject: [PATCH 70/83] Removed orphaned pretty print code for old element tree interface. #176 --- glymur/core.py | 42 ------------------------------------------ glymur/jp2box.py | 1 - 2 files changed, 43 deletions(-) diff --git a/glymur/core.py b/glymur/core.py index 30282f2..4e8f950 100644 --- a/glymur/core.py +++ b/glymur/core.py @@ -76,45 +76,3 @@ _CAPABILITIES_DISPLAY = { 1: '0', 2: '1', 3: '3'} - - -def _pretty_print_xml(xml, level=0): - """Pretty print XML data. - """ - #xml = copy.deepcopy(xml) - #_indent(xml.getroot(), level=level) - xmltext = ET.tostring(xml, encoding='utf-8').decode('utf-8') - - # Indent it a bit. - lst = [(' ' + x) for x in xmltext.split('\n')] - try: - xml = '\n'.join(lst) - return '\n{0}'.format(xml) - except UnicodeEncodeError: - # This can happen on python 2.x if the character set contains certain - # non-ascii characters. Just print out the corresponding xml char - # entities instead. - xml = u'\n'.join(lst) - text = u'\n{0}'.format(xml) - text = text.encode('ascii', 'xmlcharrefreplace') - return text - - -def _indent(elem, level=0): - """Recipe for pretty printing XML. Please see - - http://effbot.org/zone/element-lib.htm#prettyprint - """ - i = "\n" + level * " " - if len(elem): - if not elem.text or not elem.text.strip(): - elem.text = i + " " - if not elem.tail or not elem.tail.strip(): - elem.tail = i - for elem in elem: - _indent(elem, level + 1) - if not elem.tail or not elem.tail.strip(): - elem.tail = i - else: - if level and (not elem.tail or not elem.tail.strip()): - elem.tail = i diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 192763f..b4cc803 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -32,7 +32,6 @@ from .core import _COLOR_TYPE_MAP_DISPLAY from .core import SRGB, GREYSCALE, YCC from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .core import ANY_ICC_PROFILE, VENDOR_COLOR_METHOD -from .core import _pretty_print_xml from . import _uuid_io From 7828dc75e6683353cc6363d97c77bc4c61648887 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 27 Feb 2014 19:44:05 -0500 Subject: [PATCH 71/83] Updated lxml requirements, checked on Raspbian. #176 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b1332d8..5b91590 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ kwargs = {'name': 'Glymur', 'license': 'MIT', 'test_suite': 'glymur.test'} -instllrqrs = ['numpy>=1.4.1'] +instllrqrs = ['numpy>=1.4.1', 'lxml>=2.3.2'] if sys.hexversion < 0x03030000: instllrqrs.append('contextlib2>=0.4') instllrqrs.append('mock>=1.0.1') From 892715e21d6dad688992de2e204b26ff784802f0 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 27 Feb 2014 20:19:18 -0500 Subject: [PATCH 72/83] Updated docs for lxml requirement. #176 --- docs/source/changelog.rst | 1 + docs/source/detailed_installation.rst | 3 ++- docs/source/how_do_i.rst | 10 ++++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index ad4e40b..bbba98e 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -5,6 +5,7 @@ ChangeLog 0.6.0 (pending) =============== + * Added lxml requirement. * added set_printoptions, get_printoptions function * dropped support for Python 2.6, added support for Python 3.4 * dropped windows support diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index d1b3084..cfbb168 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -68,8 +68,8 @@ For python 3.3, you should install the following set of ports: * python33 * py33-numpy + * py33-lxml * py33-distribute - * py33-matplotlib (optional, for running certain tests) * py33-Pillow (optional, for running certain tests) MacPorts supplies both OpenJPEG 1.5.0 and OpenJPEG 2.0.0. @@ -85,6 +85,7 @@ packages may also need to be installed. Consult your package manager documentation or use pip. * setuptools + * python-lxml * matplotlib * pillow * contextlib2 (2.7 only) diff --git a/docs/source/how_do_i.rst b/docs/source/how_do_i.rst index fa8f098..4bde854 100644 --- a/docs/source/how_do_i.rst +++ b/docs/source/how_do_i.rst @@ -131,7 +131,6 @@ From within Python, it is as simple as printing the Jp2k object, i.e. :: - Contiguous Codestream Box (jp2c) @ (3223, 1132296) Main header: SOC marker segment @ (3231, 0) @@ -228,7 +227,6 @@ The :py:meth:`append` method can add an XML box as shown below:: >>> import shutil >>> import glymur >>> shutil.copyfile(glymur.data.nemo(), 'myfile.jp2') - >>> from xml.etree import cElementTree as ET >>> jp2 = glymur.Jp2k('myfile.jp2') >>> xmlbox = glymur.jp2box.XMLBox(filename='data.xml') >>> jp2.append(xmlbox) @@ -408,12 +406,12 @@ The example JP2 file shipped with glymur has an XMP UUID. :: . -Since the UUID data in this case is returned as an ElementTree instance, -one can use ElementTree from the standard library to access the data. -For example, to extract the **CreatorTool** attribute value, one could do the +Since the UUID data in this case is returned as an lxml ElementTree +instance, one can use lxml to access the data. For example, to +extract the **CreatorTool** attribute value, one could do the following - >>> xmp = j.box[3].data.packet + >>> xmp = j.box[3].data >>> rdf = '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}' >>> ns2 = '{http://ns.adobe.com/xap/1.0/}' >>> name = '{0}RDF/{0}Description/{1}CreatorTool'.format(rdf, ns2) From 7be9e726388bb236e58b639aec82113384748698 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 27 Feb 2014 20:43:57 -0500 Subject: [PATCH 73/83] refactored nemo xml #176 --- glymur/test/fixtures.py | 186 +++------------------------------------- 1 file changed, 10 insertions(+), 176 deletions(-) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index e7c81a3..3914173 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -181,10 +181,7 @@ def read_pgx_header(pgx_file): header = header.rstrip() return header, pos -nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) - UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - +nemo_xml = """ @@ -270,6 +267,11 @@ nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) """ +nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: +{0}""".format(nemo_xml) + SimpleRDF = """ @@ -374,91 +376,7 @@ JP2 Header Box (jp2h) @ (32, 45) UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) UUID Data: - - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - - +{0} Contiguous Codestream Box (jp2c) @ (3223, 1132296) Main header: SOC marker segment @ (3231, 0) @@ -495,7 +413,7 @@ Contiguous Codestream Box (jp2c) @ (3223, 1132296) Quantization style: no quantization, 2 guard bits Step size: [(0, 8), (0, 9), (0, 9), (0, 10)] CME marker segment @ (3305, 37) - "Created by OpenJPEG version 2.0.0"''' + "Created by OpenJPEG version 2.0.0"'''.format(nemo_xml) nemo_dump_short = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) File Type Box (ftyp) @ (12, 20) @@ -580,92 +498,8 @@ JP2 Header Box (jp2h) @ (32, 45) UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) UUID Data: - - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - - -Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" +{0} +Contiguous Codestream Box (jp2c) @ (3223, 1132296)""".format(nemo_xml) nemo_dump_no_codestream_no_xml = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) Signature: 0d0a870a From 3f3749480310e776057e2918820ff82d4e9a27f1 Mon Sep 17 00:00:00 2001 From: John Evans Date: Sat, 1 Mar 2014 12:16:42 -0500 Subject: [PATCH 74/83] Fixing indentation issues with textwrap. #176 XML had been flush left since introducing lxml, but the textwrap module fixes that easily. --- glymur/jp2box.py | 27 +++-- glymur/test/fixtures.py | 198 +++-------------------------------- glymur/test/test_printing.py | 1 + 3 files changed, 30 insertions(+), 196 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index b4cc803..5a0febf 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -20,6 +20,7 @@ import os import pprint import struct import sys +import textwrap import uuid import warnings @@ -96,10 +97,8 @@ class Jp2kBox(object): msg = Jp2kBox.__str__(self) for box in self.box: boxstr = str(box) - - # Add indentation. - strs = [('\n ' + x) for x in boxstr.split('\n')] - msg += ''.join(strs) + # Indent the child boxes to make the association clear. + msg += '\n' + textwrap.indent(boxstr, ' ') return msg @@ -889,10 +888,7 @@ class ContiguousCodestreamBox(Jp2kBox): msg += '\n Main header:' for segment in self.main_header.segment: - segstr = str(segment) - # Add indentation. - strs = [('\n ' + x) for x in segstr.split('\n')] - msg += ''.join(strs) + msg += '\n' + textwrap.indent(str(segment), ' ') return msg @@ -2562,14 +2558,14 @@ class XMLBox(Jp2kBox): if _printoptions['xml'] == False: return msg - xml = self.xml + msg += '\n' if self.xml is not None: - bstr = ET.tostring(self.xml, - encoding='utf-8', - pretty_print=True).decode('utf-8') - msg += '\n {0}'.format(bstr) + xmlstring = ET.tostring(self.xml, + encoding='utf-8', + pretty_print=True).decode('utf-8') else: - msg += '\n {0}'.format(xml) + xmlstring = 'None' + msg += textwrap.indent(xmlstring, ' ') return msg def write(self, fptr): @@ -2977,7 +2973,8 @@ class UUIDBox(Jp2kBox): xmlstring = ET.tostring(self.data, encoding='utf-8', pretty_print=True).decode('utf-8') - xmlstring = xmlstring.rstrip() + # indent it a bit + xmlstring = textwrap.indent(xmlstring.rstrip(), ' ') msg += line.format(xmlstring) elif self.uuid.bytes == b'JpgTiffExif->JP2': msg += '\n UUID Data: {0}'.format(str(self.data)) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index e7c81a3..b20331c 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -4,6 +4,7 @@ Test fixtures common to more than one test point. import os import re import sys +import textwrap import warnings import numpy as np @@ -181,10 +182,7 @@ def read_pgx_header(pgx_file): header = header.rstrip() return header, pos -nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) - UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) - UUID Data: - +nemo_xmp = """ @@ -269,6 +267,10 @@ nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) """ +nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) + UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) + UUID Data: +{0}""".format(textwrap.indent(nemo_xmp, ' ')) SimpleRDF = """ - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - - +{0} Contiguous Codestream Box (jp2c) @ (3223, 1132296) Main header: SOC marker segment @ (3231, 0) @@ -496,6 +414,7 @@ Contiguous Codestream Box (jp2c) @ (3223, 1132296) Step size: [(0, 8), (0, 9), (0, 9), (0, 10)] CME marker segment @ (3305, 37) "Created by OpenJPEG version 2.0.0"''' +nemo_dump_full = dump.format(textwrap.indent(nemo_xmp, ' ')) nemo_dump_short = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) File Type Box (ftyp) @ (12, 20) @@ -561,7 +480,7 @@ Contiguous Codestream Box (jp2c) @ (3223, 1132296) CME marker segment @ (3305, 37) "Created by OpenJPEG version 2.0.0"''' -nemo_dump_no_codestream = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) +dump = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) Signature: 0d0a870a File Type Box (ftyp) @ (12, 20) Brand: jp2 @@ -580,92 +499,9 @@ JP2 Header Box (jp2h) @ (32, 45) UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) UUID Data: - - - - - Google - 2013-02-09T14:47:53 - - - 1 - 72/1 - 72/1 - 2 - HTC - HTC Glacier - 2592 - 1456 - - - 8 - 8 - 8 - - - 2 - 3 - - - 1343036288/4294967295 - 1413044224/4294967295 - - - - - 2748779008/4294967295 - 1417339264/4294967295 - 1288490240/4294967295 - 2576980480/4294967295 - 644245120/4294967295 - 257698032/4294967295 - - - - - 1 - 2528 - 1424 - 353/100 - 0 - 0/1 - WGS-84 - 2013-02-09T14:47:53 - - - 76 - - - 0220 - 0100 - - - 1 - 2 - 3 - 0 - - - 42,20.56N - 71,5.29W - 2013-02-09T19:47:53Z - NETWORK - - - 2013-02-09T14:47:53 - - - - - Glymur - Python XMP Toolkit - - - - - - +{0} Contiguous Codestream Box (jp2c) @ (3223, 1132296)""" +nemo_dump_no_codestream = dump.format(textwrap.indent(nemo_xmp, ' ')) nemo_dump_no_codestream_no_xml = r"""JPEG 2000 Signature Box (jP ) @ (0, 12) Signature: 0d0a870a @@ -699,8 +535,8 @@ file7_rreq = r"""Reader Requirements Box (rreq) @ (44, 24) file1_xml = """XML Box (xml ) @ (36, 439) -\t -\t\t2001-11-01T13:45:00.000-06:00 -\t\tProfessional 120 Image -\t -""" + \t + \t\t2001-11-01T13:45:00.000-06:00 + \t\tProfessional 120 Image + \t + """ diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index 45b21df..923420b 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -631,6 +631,7 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(j.box[2]) actual = fake_out.getvalue().strip() + self.maxDiff = None self.assertEqual(actual, fixtures.file1_xml) @unittest.skipIf(sys.hexversion < 0x03000000, From 53f46fa0d930b87e400b9906be4254d23792fc65 Mon Sep 17 00:00:00 2001 From: John Evans Date: Sun, 2 Mar 2014 11:17:44 -0500 Subject: [PATCH 75/83] Wrapping indentation for sake of 2.7. #177 Textwrap's indent method is not available on 2.7, so we have to fake it ourselves. --- glymur/jp2box.py | 34 ++++++++++++++++++++++++++++++---- glymur/test/fixtures.py | 34 ++++++++++++++++++++++++++++++---- glymur/test/test_printing.py | 4 +++- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/glymur/jp2box.py b/glymur/jp2box.py index 5a0febf..6db6c87 100644 --- a/glymur/jp2box.py +++ b/glymur/jp2box.py @@ -98,10 +98,36 @@ class Jp2kBox(object): for box in self.box: boxstr = str(box) # Indent the child boxes to make the association clear. - msg += '\n' + textwrap.indent(boxstr, ' ') + msg += '\n' + self._indent(boxstr) return msg + def _indent(self, textstr, indent_level=4): + """ + Indent a string. + + Textwrap's indent method only exists for 3.3 or above. In 2.7 we have + to fake it. + + Parameters + ---------- + textstring : str + String to be indented. + indent_level : str + Number of spaces of indentation to add. + + Returns + ------- + indented_string : str + Possibly multi-line string indented a certain bit. + """ + if sys.hexversion >= 0x03030000: + return textwrap.indent(textstr, ' ' * indent_level) + else: + lst = [(' ' * indent_level + x) for x in textstr.split('\n')] + return '\n'.join(lst) + + def _write_superbox(self, fptr): """Write a superbox. @@ -888,7 +914,7 @@ class ContiguousCodestreamBox(Jp2kBox): msg += '\n Main header:' for segment in self.main_header.segment: - msg += '\n' + textwrap.indent(str(segment), ' ') + msg += '\n' + self._indent(str(segment), indent_level=8) return msg @@ -2565,7 +2591,7 @@ class XMLBox(Jp2kBox): pretty_print=True).decode('utf-8') else: xmlstring = 'None' - msg += textwrap.indent(xmlstring, ' ') + msg += self._indent(xmlstring) return msg def write(self, fptr): @@ -2974,7 +3000,7 @@ class UUIDBox(Jp2kBox): encoding='utf-8', pretty_print=True).decode('utf-8') # indent it a bit - xmlstring = textwrap.indent(xmlstring.rstrip(), ' ') + xmlstring = self._indent(xmlstring.rstrip()) msg += line.format(xmlstring) elif self.uuid.bytes == b'JpgTiffExif->JP2': msg += '\n UUID Data: {0}'.format(str(self.data)) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index fe0777e..666fb6b 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -45,6 +45,32 @@ except: raise +def _indent(textstr): + """ + Indent a string. + + Textwrap's indent method only exists for 3.3 or above. In 2.7 we have + to fake it. + + Parameters + ---------- + textstring : str + String to be indented. + indent_level : str + Number of spaces of indentation to add. + + Returns + ------- + indented_string : str + Possibly multi-line string indented a certain bit. + """ + if sys.hexversion >= 0x03030000: + return textwrap.indent(textstr, ' ') + else: + lst = [(' ' + x) for x in textstr.split('\n')] + return '\n'.join(lst) + + def opj_data_file(relative_file_name): """Compact way of forming a full filename from OpenJPEG's test suite.""" jfile = os.path.join(OPJ_DATA_ROOT, relative_file_name) @@ -270,12 +296,12 @@ nemo_xmp = """ nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) UUID Data: -{0}""".format(textwrap.indent(nemo_xmp, ' ')) +{0}""".format(_indent(nemo_xmp)) nemo_xmp_box = """UUID Box (uuid) @ (77, 3146) UUID: be7acfcb-97a9-42e8-9c71-999491e3afac (XMP) UUID Data: -{0}""".format(textwrap.indent(nemo_xmp, ' ')) +{0}""".format(_indent(nemo_xmp)) SimpleRDF = """ Date: Wed, 5 Mar 2014 07:04:10 -0500 Subject: [PATCH 76/83] Verified through r2366. #139 423 of 457 glymur tests passing, 34 skips 21 of 214 openjpeg tests failed --- docs/source/detailed_installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index cfbb168..ae0ce08 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -13,7 +13,7 @@ both read and write JPEG 2000 files, but you may wish to install version 2.0 or the 2.0+ version from OpenJPEG's development trunk for better performance. If you do that, you should compile it as a shared library (named *openjp2* instead of *openjpeg*) from the developmental source that you can retrieve -via subversion. As of this time of writing, svn revision r2354 works. +via subversion. As of this time of writing, svn revision r2366 works. You should also download the test data for the purpose of configuring and running OpenJPEG's test suite, check their instructions for all this. You should set the **OPJ_DATA_ROOT** environment variable for the purpose From b6a17d60f1a4e5bbf388c30d0c290fb749381c4d Mon Sep 17 00:00:00 2001 From: John Evans Date: Wed, 5 Mar 2014 21:17:35 -0500 Subject: [PATCH 77/83] Starting to implement cinema2k support. #139 Need to add scikit-image with freeimage backend in order to read 16-bit TIFFs. --- glymur/jp2k.py | 61 ++++++++++++++++++++++++++++- glymur/test/test_conformance.py | 54 ++++++++++++++++--------- glymur/test/test_opj_suite_write.py | 10 +++++ 3 files changed, 106 insertions(+), 19 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 6db0198..04fa23e 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -219,6 +219,63 @@ class Jp2k(Jp2kBox): cparams.tcp_numlayers = 1 cparams.cp_disto_alloc = 1 + if 'cinema2K' in kwargs: + cparams.cp_rsiz = kwargs['cinema2K'] + # No tiling + cparams.tile_size_on = opj2.FALSE + cparams.cp_tdx = 1 + cparams.cp_tdy = 1 + + # One tile part for each component. + cparams.tp_cflag = ord('C') + cparams.tp_on = 1 + + # tile and image shall be as (0,0) + cparams.cp_tx0 = 0 + cparams.cp_ty0 = 0 + cparams.image_offset_x0 = 0 + cparams.image_offset_y0 = 0 + + # Codeblock size = 32 * 32 + cparams.cblockw_init = 32 + cparams.cblockh_init = 32 + + # code block style, no mode switch enabled. + cparams.mode = 0 + + # no ROI + cparams.roi_compno = -1 + + # no subsampling + cparams.subsampling_dx = 1 + cparams.subsampling_dy = 1 + + # 9-7 transform + cparams.irreversible = 1 + + # number of layers + if cparams.tcp_layers > 1: + # TODO: warning or error + cparams.tcp_numlayers = 1 + + if cparams.numresolution > 6: + # TODO: warning or error + cparams.numresolution = 6 + + # precincts + cparams.csty |= 0x01 + cparams.res_spec = cparams.numresolution - 1 + for j in range(cparams.res_spec): + cparams.prcw_init[i] = 256 + cparams.prch_init[i] = 256 + + # Progression order shall be CPRL + cparams.prog_order = PROGRESSION_ORDER['CPRL'] + + # progression order changes not allowed for 2K + cparams.numpocs = 0 + return cparams + if 'cbsize' in kwargs: cparams.cblockw_init = kwargs['cbsize'][1] cparams.cblockh_init = kwargs['cbsize'][0] @@ -340,6 +397,8 @@ class Jp2k(Jp2kBox): Image data to be written to file. cbsize : tuple, optional Code block size (DY, DX). + cinema2K : int, optional + either 24 or 48 colorspace : str, optional Either 'rgb' or 'gray'. cratios : iterable @@ -473,7 +532,7 @@ class Jp2k(Jp2kBox): def _write_openjp2(self, img_array, verbose=False, **kwargs): """ - Write JPEG 2000 file using OpenJPEG 1.5 interface. + Write JPEG 2000 file using OpenJPEG 2.0 interface. """ cparams, colorspace = self._process_write_inputs(img_array, **kwargs) diff --git a/glymur/test/test_conformance.py b/glymur/test/test_conformance.py index 8f4496d..765f2f6 100644 --- a/glymur/test/test_conformance.py +++ b/glymur/test/test_conformance.py @@ -11,6 +11,7 @@ import os from os.path import join import re import sys +import tempfile import unittest import glymur @@ -27,6 +28,41 @@ except KeyError: OPJ_DATA_ROOT = None +@unittest.skipIf(sys.hexversion < 0x03020000, + "Requires features introduced in 3.2 (assertWarns)") +class TestSuiteConformance(unittest.TestCase): + """Test suite for conformance.""" + + def setUp(self): + self.j2kfile = glymur.data.goodstuff() + + def tearDown(self): + pass + + @unittest.skipIf(re.match(r"""1\.[0123]""", + glymur.version.openjpeg_version) is not None, + "Needs 1.3+ to catch this.") + def test_truncated_eoc(self): + """Has one byte shaved off of EOC marker.""" + with open(self.j2kfile, 'rb') as ifile: + data = ifile.read() + with tempfile.NamedTemporaryFile(suffix='.j2k') as ofile: + ofile.write(data[:-1]) + ofile.flush() + + j2k = Jp2k(ofile.name) + with self.assertWarns(UserWarning): + codestream = j2k.get_codestream(header_only=False) + + # The last segment is truncated, so there should not be an EOC + # marker. + self.assertNotEqual(codestream.segment[-1].marker_id, 'EOC') + + # The codestream is not as long as claimed. + with self.assertRaises(OSError): + j2k.read(rlevel=-1) + + @unittest.skipIf(FORMAT_CORPUS_DATA_ROOT is None, "FORMAT_CORPUS_DATA_ROOT environment variable not set") @unittest.skipIf(sys.hexversion < 0x03020000, @@ -34,24 +70,6 @@ except KeyError: class TestSuiteFormatCorpus(unittest.TestCase): """Test suite for files in format corpus repository.""" - @unittest.skipIf(re.match(r"""1\.[0123]""", - glymur.version.openjpeg_version) is not None, - "Needs 1.3+ to catch this.") - def test_balloon_trunc1(self): - """Has one byte shaved off of EOC marker.""" - jfile = os.path.join(FORMAT_CORPUS_DATA_ROOT, - 'jp2k-test/byteCorruption/balloon_trunc1.jp2') - j2k = Jp2k(jfile) - with self.assertWarns(UserWarning): - codestream = j2k.get_codestream(header_only=False) - - # The last segment is truncated, so there should not be an EOC marker. - self.assertNotEqual(codestream.segment[-1].marker_id, 'EOC') - - # The codestream is not as long as claimed. - with self.assertRaises(OSError): - j2k.read(rlevel=-1) - @unittest.skipIf(re.match(r"""1\.[01234]""", glymur.version.openjpeg_version) is not None, "Needs 1.4+ to catch this.") diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index 22a99e3..07d623f 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -38,6 +38,16 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass + @unittest.skip("Cannot read input image using PILLOW???") + def test_NR_ENC_X_4_2K_24_185_CBR_WB_000_tif_15_encode(self): + relfile = 'input/nonregression/X_4_2K_24_185_CBR_WB_000.tif' + import pdb; pdb.set_trace() + infile = opj_data_file(relfile) + data = read_image(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, 'cinema2K', 24) + def test_NR_ENC_Bretagne1_ppm_1_encode(self): """NR-ENC-Bretagne1.ppm-1-encode""" infile = opj_data_file('input/nonregression/Bretagne1.ppm') From afa611ae543fb166d5f1f2958579d5ae4448185f Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 6 Mar 2014 07:10:29 -0500 Subject: [PATCH 78/83] Can write and parse Cinema2K image. #139 Using scikit-image to read the test data. Testing still not complete. --- glymur/core.py | 20 ++++++++ glymur/jp2k.py | 43 ++++++++++++++--- glymur/test/test_opj_suite_write.py | 74 +++++++++++++++++++++++++++-- 3 files changed, 127 insertions(+), 10 deletions(-) diff --git a/glymur/core.py b/glymur/core.py index 4e8f950..95e1cbf 100644 --- a/glymur/core.py +++ b/glymur/core.py @@ -10,6 +10,26 @@ RPCL = 2 PCRL = 3 CPRL = 4 +STD = 0 +CINEMA2K = 3 +CINEMA4K = 4 + +RSIZ = { + 'STD': STD, + 'CINEMA2K': CINEMA2K, + 'CINEMA4K': CINEMA4K} + +OFF = 0 +CINEMA2K_24 = 1 +CINEMA2K_48 = 2 +CINEMA4K_24 = 3 + +CINEMA_MODE = { + 'off': OFF, + 'cinema2k_24': CINEMA2K_24, + 'cinema2k_48': CINEMA2K_48, + 'cinema4k_24': CINEMA4K_24, } + PROGRESSION_ORDER = { 'LRCP': LRCP, 'RLCP': RLCP, diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 04fa23e..890c2fd 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -28,7 +28,7 @@ import numpy as np from .codestream import Codestream from .core import SRGB, GREYSCALE -from .core import PROGRESSION_ORDER +from .core import PROGRESSION_ORDER, RSIZ, CINEMA_MODE from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from .jp2box import Jp2kBox from .jp2box import JPEG2000SignatureBox, FileTypeBox, JP2HeaderBox @@ -39,6 +39,8 @@ from .lib import openjp2 as opj2 from . import version from .lib import c as libc +CINEMA_24_CS = 1302083 + JP2_IDS = ['colr', 'cdef', 'cmap', 'jp2c', 'ftyp', 'ihdr', 'jp2h', 'jP ', 'pclr', 'res ', 'resc', 'resd', 'xml ', 'ulst', 'uinf', 'url ', 'uuid'] @@ -220,14 +222,17 @@ class Jp2k(Jp2kBox): cparams.cp_disto_alloc = 1 if 'cinema2K' in kwargs: - cparams.cp_rsiz = kwargs['cinema2K'] + # TODO: error if either 24 or 48 + cparams.cp_cinema = kwargs['cinema2K'] + + cparams.cp_rsiz = RSIZ['CINEMA2K'] # No tiling cparams.tile_size_on = opj2.FALSE cparams.cp_tdx = 1 cparams.cp_tdy = 1 # One tile part for each component. - cparams.tp_cflag = ord('C') + cparams.tp_flag = ord('C') cparams.tp_on = 1 # tile and image shall be as (0,0) @@ -254,7 +259,7 @@ class Jp2k(Jp2kBox): cparams.irreversible = 1 # number of layers - if cparams.tcp_layers > 1: + if cparams.tcp_numlayers > 1: # TODO: warning or error cparams.tcp_numlayers = 1 @@ -266,8 +271,8 @@ class Jp2k(Jp2kBox): cparams.csty |= 0x01 cparams.res_spec = cparams.numresolution - 1 for j in range(cparams.res_spec): - cparams.prcw_init[i] = 256 - cparams.prch_init[i] = 256 + cparams.prcw_init[j] = 256 + cparams.prch_init[j] = 256 # Progression order shall be CPRL cparams.prog_order = PROGRESSION_ORDER['CPRL'] @@ -530,6 +535,29 @@ class Jp2k(Jp2kBox): self.parse() + def _set_cinema_rate(self, cparams, image): + max_rate = 0 + temp_rate = 0 + cparams.cp_disto_alloc = 1 + + if cparams.cp_cinema in [CINEMA_MODE['cinema2k_24'], + CINEMA_MODE['cinema2k_48']]: + num_pixels = image.contents.comps[0].w * image.contents.comps[0].h + rate_numerator = num_pixels * image.contents.comps[0].prec + max_rate = rate_numerator / (CINEMA_24_CS * 8 * num_pixels) + if cparams.tcp_rates[0] == 0: + cparams.tcp_rates[0] = max_rate + else: + temp_rate = rate_numerator / (cparams.tcp_rates[0] * 8 * num_pixels) + if temp_rate > CINEMA_24_CS: + # TODO warning, reset + cparams.tcp_rates[0] = max_rate + else: + # TODO warning + pass + + cparams.max_comp_size = COMP_24_CS + def _write_openjp2(self, img_array, verbose=False, **kwargs): """ Write JPEG 2000 file using OpenJPEG 2.0 interface. @@ -549,6 +577,9 @@ class Jp2k(Jp2kBox): _populate_image_struct(cparams, image, img_array) + if 'cinema2K' in kwargs: + self._set_cinema_rate(cparams, image) + codec = opj2.create_compress(cparams.codec_fmt) stack.callback(opj2.destroy_codec, codec) diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index 07d623f..335202d 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -12,6 +12,13 @@ import sys import tempfile import unittest +try: + import skimage.io + skimage.io.use_plugin('freeimage', 'imread') + _HAS_SKIMAGE_FREEIMAGE_SUPPORT = True +except ImportError: + _HAS_SKIMAGE_FREEIMAGE_SUPPORT = False + from .fixtures import read_image, NO_READ_BACKEND, NO_READ_BACKEND_MSG from .fixtures import OPJ_DATA_ROOT, opj_data_file @@ -38,15 +45,74 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass - @unittest.skip("Cannot read input image using PILLOW???") + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_4_2K_24_185_CBR_WB_000_tif_15_encode(self): relfile = 'input/nonregression/X_4_2K_24_185_CBR_WB_000.tif' - import pdb; pdb.set_trace() infile = opj_data_file(relfile) - data = read_image(infile) + data = skimage.io.imread(infile) with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: j = Jp2k(tfile.name, 'wb') - j.write(data, 'cinema2K', 24) + j.write(data, cinema2K=24) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (1998, 1080)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (1998, 1080)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CRLP) + self.assertEqual(codestream.segment[2].layers, 3) # layers = 3 + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (64, 64)) # cblksz + # Selective arithmetic coding bypass + self.assertFalse(codestream.segment[2].spcod[7] & 0x01) + # Reset context probabilities + self.assertFalse(codestream.segment[2].spcod[7] & 0x02) + # Termination on each coding pass + self.assertFalse(codestream.segment[2].spcod[7] & 0x04) + # Vertically causal context + self.assertFalse(codestream.segment[2].spcod[7] & 0x08) + # Predictable termination + self.assertFalse(codestream.segment[2].spcod[7] & 0x0010) + # Segmentation symbols + self.assertFalse(codestream.segment[2].spcod[7] & 0x0020) + self.assertEqual(codestream.segment[2].spcod[8], + glymur.core.WAVELET_XFORM_5X3_REVERSIBLE) + self.assertEqual(len(codestream.segment[2].spcod), 9) + + + + def test_NR_ENC_Bretagne1_ppm_1_encode(self): """NR-ENC-Bretagne1.ppm-1-encode""" From 3bad8a4b2e5444484e959d74b966845d7221a39c Mon Sep 17 00:00:00 2001 From: John Evans Date: Thu, 6 Mar 2014 09:20:21 -0500 Subject: [PATCH 79/83] Added negative test for cinema2k frame rate not 24 or 48. #139 Some refactoring of cinema2k code. --- glymur/jp2k.py | 133 +++++++++++++++------------- glymur/test/fixtures.py | 2 +- glymur/test/test_opj_suite_write.py | 16 +++- glymur/test/test_printing.py | 5 +- 4 files changed, 88 insertions(+), 68 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 890c2fd..255e2be 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -155,6 +155,73 @@ class Jp2k(Jp2kBox): msg += "profile if the file type box brand is 'jp2 '." warnings.warn(msg) + def _set_cinema_params(self, cparams, fps): + """Populate compression parameters structure for cinema2K. + + Parameters + ---------- + fps : int + Frames per second, should be either 24 or 48. + """ + if fps not in [24, 48]: + raise IOError('Cinema2K frame rate must be either 24 or 48.') + cparams.cp_cinema = fps + + cparams.cp_rsiz = RSIZ['CINEMA2K'] + # No tiling + cparams.tile_size_on = opj2.FALSE + cparams.cp_tdx = 1 + cparams.cp_tdy = 1 + + # One tile part for each component. + cparams.tp_flag = ord('C') + cparams.tp_on = 1 + + # tile and image shall be as (0,0) + cparams.cp_tx0 = 0 + cparams.cp_ty0 = 0 + cparams.image_offset_x0 = 0 + cparams.image_offset_y0 = 0 + + # Codeblock size = 32 * 32 + cparams.cblockw_init = 32 + cparams.cblockh_init = 32 + + # code block style, no mode switch enabled. + cparams.mode = 0 + + # no ROI + cparams.roi_compno = -1 + + # no subsampling + cparams.subsampling_dx = 1 + cparams.subsampling_dy = 1 + + # 9-7 transform + cparams.irreversible = 1 + + # number of layers + if cparams.tcp_numlayers > 1: + # TODO: warning or error + cparams.tcp_numlayers = 1 + + if cparams.numresolution > 6: + # TODO: warning or error + cparams.numresolution = 6 + + # precincts + cparams.csty |= 0x01 + cparams.res_spec = cparams.numresolution - 1 + for j in range(cparams.res_spec): + cparams.prcw_init[j] = 256 + cparams.prch_init[j] = 256 + + # Progression order shall be CPRL + cparams.prog_order = PROGRESSION_ORDER['CPRL'] + + # progression order changes not allowed for 2K + cparams.numpocs = 0 + def _populate_cparams(self, **kwargs): """Populate compression parameters structure from input arguments. @@ -221,64 +288,8 @@ class Jp2k(Jp2kBox): cparams.tcp_numlayers = 1 cparams.cp_disto_alloc = 1 - if 'cinema2K' in kwargs: - # TODO: error if either 24 or 48 - cparams.cp_cinema = kwargs['cinema2K'] - - cparams.cp_rsiz = RSIZ['CINEMA2K'] - # No tiling - cparams.tile_size_on = opj2.FALSE - cparams.cp_tdx = 1 - cparams.cp_tdy = 1 - - # One tile part for each component. - cparams.tp_flag = ord('C') - cparams.tp_on = 1 - - # tile and image shall be as (0,0) - cparams.cp_tx0 = 0 - cparams.cp_ty0 = 0 - cparams.image_offset_x0 = 0 - cparams.image_offset_y0 = 0 - - # Codeblock size = 32 * 32 - cparams.cblockw_init = 32 - cparams.cblockh_init = 32 - - # code block style, no mode switch enabled. - cparams.mode = 0 - - # no ROI - cparams.roi_compno = -1 - - # no subsampling - cparams.subsampling_dx = 1 - cparams.subsampling_dy = 1 - - # 9-7 transform - cparams.irreversible = 1 - - # number of layers - if cparams.tcp_numlayers > 1: - # TODO: warning or error - cparams.tcp_numlayers = 1 - - if cparams.numresolution > 6: - # TODO: warning or error - cparams.numresolution = 6 - - # precincts - cparams.csty |= 0x01 - cparams.res_spec = cparams.numresolution - 1 - for j in range(cparams.res_spec): - cparams.prcw_init[j] = 256 - cparams.prch_init[j] = 256 - - # Progression order shall be CPRL - cparams.prog_order = PROGRESSION_ORDER['CPRL'] - - # progression order changes not allowed for 2K - cparams.numpocs = 0 + if 'cinema2k' in kwargs: + self._set_cinema_params(cparams, kwargs['cinema2k']) return cparams if 'cbsize' in kwargs: @@ -402,8 +413,8 @@ class Jp2k(Jp2kBox): Image data to be written to file. cbsize : tuple, optional Code block size (DY, DX). - cinema2K : int, optional - either 24 or 48 + cinema2k : int, optional + frames per second, either 24 or 48 colorspace : str, optional Either 'rgb' or 'gray'. cratios : iterable @@ -577,7 +588,7 @@ class Jp2k(Jp2kBox): _populate_image_struct(cparams, image, img_array) - if 'cinema2K' in kwargs: + if 'cinema2k' in kwargs: self._set_cinema_rate(cparams, image) codec = opj2.create_compress(cparams.codec_fmt) diff --git a/glymur/test/fixtures.py b/glymur/test/fixtures.py index 666fb6b..7e4d9e8 100644 --- a/glymur/test/fixtures.py +++ b/glymur/test/fixtures.py @@ -22,7 +22,7 @@ try: HAS_PYTHON_XMP_TOOLKIT = True else: HAS_PYTHON_XMP_TOOLKIT = False -except ImportError: +except: HAS_PYTHON_XMP_TOOLKIT = False # Need to know of the libopenjp2 version is the official 2.0.0 release and NOT diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index 335202d..21c2578 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -16,7 +16,7 @@ try: import skimage.io skimage.io.use_plugin('freeimage', 'imread') _HAS_SKIMAGE_FREEIMAGE_SUPPORT = True -except ImportError: +except ((ImportError, RuntimeError)): _HAS_SKIMAGE_FREEIMAGE_SUPPORT = False from .fixtures import read_image, NO_READ_BACKEND, NO_READ_BACKEND_MSG @@ -25,7 +25,6 @@ from .fixtures import OPJ_DATA_ROOT, opj_data_file from glymur import Jp2k import glymur - @unittest.skipIf(os.name == "nt", "no write support on windows, period") @unittest.skipIf(re.match(r"""1\.[01234]\.\d""", glymur.version.openjpeg_version) is not None, @@ -53,7 +52,7 @@ class TestSuiteWrite(unittest.TestCase): data = skimage.io.imread(infile) with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: j = Jp2k(tfile.name, 'wb') - j.write(data, cinema2K=24) + j.write(data, cinema2k=24) codestream = j.get_codestream() @@ -111,7 +110,16 @@ class TestSuiteWrite(unittest.TestCase): self.assertEqual(len(codestream.segment[2].spcod), 9) - + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_cinema2k_bad_frame_rate(self): + relfile = 'input/nonregression/X_4_2K_24_185_CBR_WB_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + with self.assertRaises(IOError): + j.write(data, cinema2k=36) def test_NR_ENC_Bretagne1_ppm_1_encode(self): diff --git a/glymur/test/test_printing.py b/glymur/test/test_printing.py index f90ade3..c59d73f 100644 --- a/glymur/test/test_printing.py +++ b/glymur/test/test_printing.py @@ -133,7 +133,6 @@ class TestPrinting(unittest.TestCase): lst = lst[1:] actual = '\n'.join(lst) expected = fixtures.nemo_dump_no_xml - self.maxDiff = None self.assertEqual(actual, expected) def test_printoptions_short(self): @@ -745,7 +744,9 @@ class TestPrinting(unittest.TestCase): with patch('sys.stdout', new=StringIO()) as fake_out: print(j.box[2]) actual = fake_out.getvalue().strip() - self.assertEqual(actual, fixtures.file7_rreq) + self.maxDiff = None + expected = fixtures.file7_rreq + self.assertEqual(actual, expected) @unittest.skipIf(OPJ_DATA_ROOT is None, "OPJ_DATA_ROOT environment variable not set") From 80326718a595fbfad9f93b6d8975801cd98908a2 Mon Sep 17 00:00:00 2001 From: jevans Date: Thu, 6 Mar 2014 21:13:40 -0500 Subject: [PATCH 80/83] Finished Cinema2K upstream tests. #139 --- CHANGES.txt | 3 +- docs/source/changelog.rst | 3 +- glymur/jp2k.py | 25 ++++-- glymur/test/test_opj_suite_write.py | 123 ++++++++++++++++++++++++---- 4 files changed, 127 insertions(+), 27 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 24d9616..c906d29 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ -Feb 09, 2014 - Changed constructor for ChannelDefinition box. Removed support +Mar 06, 2014 - Added Cinema2K write support. + Changed constructor for ChannelDefinition box. Removed support for Python 2.6. Added write support for JP2 UUID, DataEntryURL, Palette and Component Mapping boxes, JPX Association, NumberList and DataReference boxes. Added read support for JPX free, diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index bbba98e..320ff0f 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -5,10 +5,11 @@ ChangeLog 0.6.0 (pending) =============== + * Added Cinema2K write support. * Added lxml requirement. * added set_printoptions, get_printoptions function * dropped support for Python 2.6, added support for Python 3.4 - * dropped windows support + * dropped windows support (it might work, it might not, I don't much care) * added write support for JP2 UUID, dataEntryURL, palette, and component mapping boxes * added read/write support for JPX free, number list, and data reference boxes * Added read support for JPX fragment list and fragment table boxes diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 255e2be..627082b 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -163,9 +163,12 @@ class Jp2k(Jp2kBox): fps : int Frames per second, should be either 24 or 48. """ - if fps not in [24, 48]: + if fps == 24: + cparams.cp_cinema = CINEMA_MODE['cinema2k_24'] + elif fps == 48: + cparams.cp_cinema = CINEMA_MODE['cinema2k_48'] + else: raise IOError('Cinema2K frame rate must be either 24 or 48.') - cparams.cp_cinema = fps cparams.cp_rsiz = RSIZ['CINEMA2K'] # No tiling @@ -205,7 +208,7 @@ class Jp2k(Jp2kBox): # TODO: warning or error cparams.tcp_numlayers = 1 - if cparams.numresolution > 6: + if cparams.numresolution > 6: #TODO only for cinema2k # TODO: warning or error cparams.numresolution = 6 @@ -219,7 +222,7 @@ class Jp2k(Jp2kBox): # Progression order shall be CPRL cparams.prog_order = PROGRESSION_ORDER['CPRL'] - # progression order changes not allowed for 2K + # progression order changes not allowed for 2K # TODO not for 4K: cparams.numpocs = 0 def _populate_cparams(self, **kwargs): @@ -554,8 +557,12 @@ class Jp2k(Jp2kBox): if cparams.cp_cinema in [CINEMA_MODE['cinema2k_24'], CINEMA_MODE['cinema2k_48']]: num_pixels = image.contents.comps[0].w * image.contents.comps[0].h - rate_numerator = num_pixels * image.contents.comps[0].prec - max_rate = rate_numerator / (CINEMA_24_CS * 8 * num_pixels) + num_samples = num_pixels * image.contents.numcomps + rate_numerator = num_samples * image.contents.comps[0].prec + rate_denominator = CINEMA_24_CS * 8 + rate_denominator *= image.contents.comps[0].dx + rate_denominator *= image.contents.comps[0].dy + max_rate = rate_numerator / rate_denominator if cparams.tcp_rates[0] == 0: cparams.tcp_rates[0] = max_rate else: @@ -567,7 +574,7 @@ class Jp2k(Jp2kBox): # TODO warning pass - cparams.max_comp_size = COMP_24_CS + cparams.max_comp_size = CINEMA_24_CS def _write_openjp2(self, img_array, verbose=False, **kwargs): """ @@ -1608,6 +1615,10 @@ def _populate_image_struct(cparams, image, imgdata): # Stage the image data to the openjpeg data structure. for k in range(0, num_comps): + if cparams.cp_cinema: + image.contents.comps[k].prec = 12 + image.contents.comps[k].bpp = 12 + layer = np.ascontiguousarray(imgdata[:, :, k], dtype=np.int32) dest = image.contents.comps[k].data src = layer.ctypes.data diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index 21c2578..ca3bc83 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -44,6 +44,108 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_X_6_2K_24_FULL_CBR_CIRCLE_000_tif_17_encode(self): + relfile = 'input/nonregression/X_6_2K_24_FULL_CBR_CIRCLE_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema2k=24) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (2048, 1080)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (2048, 1080)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_X_5_2K_24_235_CBR_STEM24_000_tif_16_encode(self): + relfile = 'input/nonregression/X_5_2K_24_235_CBR_STEM24_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema2k=24) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (2048, 857)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (2048, 857)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_4_2K_24_185_CBR_WB_000_tif_15_encode(self): @@ -87,27 +189,12 @@ class TestSuiteWrite(unittest.TestCase): # COD: Coding style default self.assertFalse(codestream.segment[2].scod & 2) # no sop self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CRLP) - self.assertEqual(codestream.segment[2].layers, 3) # layers = 3 + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) self.assertEqual(codestream.segment[2].spcod[3], 1) # mct self.assertEqual(codestream.segment[2].spcod[4], 5) # levels self.assertEqual(tuple(codestream.segment[2].code_block_size), - (64, 64)) # cblksz - # Selective arithmetic coding bypass - self.assertFalse(codestream.segment[2].spcod[7] & 0x01) - # Reset context probabilities - self.assertFalse(codestream.segment[2].spcod[7] & 0x02) - # Termination on each coding pass - self.assertFalse(codestream.segment[2].spcod[7] & 0x04) - # Vertically causal context - self.assertFalse(codestream.segment[2].spcod[7] & 0x08) - # Predictable termination - self.assertFalse(codestream.segment[2].spcod[7] & 0x0010) - # Segmentation symbols - self.assertFalse(codestream.segment[2].spcod[7] & 0x0020) - self.assertEqual(codestream.segment[2].spcod[8], - glymur.core.WAVELET_XFORM_5X3_REVERSIBLE) - self.assertEqual(len(codestream.segment[2].spcod), 9) + (32, 32)) # cblksz @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, From 9a74109501fa89cc2daec8ceba8a928c6b30677e Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 7 Mar 2014 07:00:02 -0500 Subject: [PATCH 81/83] Added cinema2k/48 support. #139 --- glymur/jp2k.py | 39 ++++++- glymur/test/test_opj_suite_write.py | 153 ++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 2 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 627082b..1d9e1b2 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -39,7 +39,15 @@ from .lib import openjp2 as opj2 from . import version from .lib import c as libc +# Codestream lengths for 24fps, 48fps CINEMA_24_CS = 1302083 +CINEMA_48_CS = 651041 + +# Maximum size per color components for 2K and 4K at 24 fps +COMP_24_CS = 1041666 + +# Maximum size per color components for 2K at 48 fps +COMP_48_CS = 520833 JP2_IDS = ['colr', 'cdef', 'cmap', 'jp2c', 'ftyp', 'ihdr', 'jp2h', 'jP ', 'pclr', 'res ', 'resc', 'resd', 'xml ', 'ulst', 'uinf', 'url ', @@ -295,6 +303,10 @@ class Jp2k(Jp2kBox): self._set_cinema_params(cparams, kwargs['cinema2k']) return cparams + if 'cinema4k' in kwargs: + self._set_cinema_params(cparams, kwargs['cinema2k']) + return cparams + if 'cbsize' in kwargs: cparams.cblockw_init = kwargs['cbsize'][1] cparams.cblockh_init = kwargs['cbsize'][0] @@ -555,7 +567,7 @@ class Jp2k(Jp2kBox): cparams.cp_disto_alloc = 1 if cparams.cp_cinema in [CINEMA_MODE['cinema2k_24'], - CINEMA_MODE['cinema2k_48']]: + CINEMA_MODE['cinema4k_24']]: num_pixels = image.contents.comps[0].w * image.contents.comps[0].h num_samples = num_pixels * image.contents.numcomps rate_numerator = num_samples * image.contents.comps[0].prec @@ -574,7 +586,28 @@ class Jp2k(Jp2kBox): # TODO warning pass - cparams.max_comp_size = CINEMA_24_CS + cparams.max_comp_size = COMP_24_CS + + else: + num_pixels = image.contents.comps[0].w * image.contents.comps[0].h + num_samples = num_pixels * image.contents.numcomps + rate_numerator = num_samples * image.contents.comps[0].prec + rate_denominator = CINEMA_48_CS * 8 + rate_denominator *= image.contents.comps[0].dx + rate_denominator *= image.contents.comps[0].dy + max_rate = rate_numerator / rate_denominator + if cparams.tcp_rates[0] == 0: + cparams.tcp_rates[0] = max_rate + else: + temp_rate = rate_numerator / (cparams.tcp_rates[0] * 8 * num_pixels) + if temp_rate > CINEMA_48_CS: + # TODO warning, reset + cparams.tcp_rates[0] = max_rate + else: + # TODO warning + pass + + cparams.max_comp_size = COMP_48_CS def _write_openjp2(self, img_array, verbose=False, **kwargs): """ @@ -597,6 +630,8 @@ class Jp2k(Jp2kBox): if 'cinema2k' in kwargs: self._set_cinema_rate(cparams, image) + if 'cinema4k' in kwargs: + self._set_cinema_rate(cparams, image) codec = opj2.create_compress(cparams.codec_fmt) stack.callback(opj2.destroy_codec, codec) diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index ca3bc83..d0e449e 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -44,6 +44,108 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_X_5_2K_24_235_CBR_STEM24_000_tif_19_encode(self): + relfile = 'input/nonregression/X_5_2K_24_235_CBR_STEM24_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema2k=48) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (2048, 857)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (2048, 857)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_X_6_2K_24_FULL_CBR_CIRCLE_000_tif_20_encode(self): + relfile = 'input/nonregression/X_6_2K_24_FULL_CBR_CIRCLE_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema2k=48) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (2048, 1080)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (2048, 1080)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_6_2K_24_FULL_CBR_CIRCLE_000_tif_17_encode(self): @@ -146,6 +248,57 @@ class TestSuiteWrite(unittest.TestCase): (32, 32)) # cblksz + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_X_4_2K_24_185_CBR_WB_000_tif_18_encode(self): + relfile = 'input/nonregression/X_4_2K_24_185_CBR_WB_000.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema2k=48) + + codestream = j.get_codestream() + + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + (1998, 1080)) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + (1998, 1080)) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_4_2K_24_185_CBR_WB_000_tif_15_encode(self): From af4b6e64bfc70e6c97c7cceda509dd2685c25b68 Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 7 Mar 2014 08:40:14 -0500 Subject: [PATCH 82/83] Refactoring. #139 --- glymur/jp2k.py | 22 +-- glymur/test/test_opj_suite_write.py | 276 +++++----------------------- 2 files changed, 56 insertions(+), 242 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 1d9e1b2..00d3d24 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -566,15 +566,15 @@ class Jp2k(Jp2kBox): temp_rate = 0 cparams.cp_disto_alloc = 1 + num_pixels = image.contents.comps[0].w * image.contents.comps[0].h + num_samples = num_pixels * image.contents.numcomps + rate_numerator = num_samples * image.contents.comps[0].prec + rate_denominator = 8 * image.contents.comps[0].dx + rate_denominator *= image.contents.comps[0].dy + if cparams.cp_cinema in [CINEMA_MODE['cinema2k_24'], CINEMA_MODE['cinema4k_24']]: - num_pixels = image.contents.comps[0].w * image.contents.comps[0].h - num_samples = num_pixels * image.contents.numcomps - rate_numerator = num_samples * image.contents.comps[0].prec - rate_denominator = CINEMA_24_CS * 8 - rate_denominator *= image.contents.comps[0].dx - rate_denominator *= image.contents.comps[0].dy - max_rate = rate_numerator / rate_denominator + max_rate = rate_numerator / (rate_denominator * CINEMA_24_CS) if cparams.tcp_rates[0] == 0: cparams.tcp_rates[0] = max_rate else: @@ -589,13 +589,7 @@ class Jp2k(Jp2kBox): cparams.max_comp_size = COMP_24_CS else: - num_pixels = image.contents.comps[0].w * image.contents.comps[0].h - num_samples = num_pixels * image.contents.numcomps - rate_numerator = num_samples * image.contents.comps[0].prec - rate_denominator = CINEMA_48_CS * 8 - rate_denominator *= image.contents.comps[0].dx - rate_denominator *= image.contents.comps[0].dy - max_rate = rate_numerator / rate_denominator + max_rate = rate_numerator / (rate_denominator * CINEMA_48_CS) if cparams.tcp_rates[0] == 0: cparams.tcp_rates[0] = max_rate else: diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index d0e449e..87a6fc4 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -44,6 +44,48 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass + def check_cinema2k_codestream(self, codestream, image_size): + """Common out for cinema2k tests.""" + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 3) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + image_size) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + image_size) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_5_2K_24_235_CBR_STEM24_000_tif_19_encode(self): @@ -55,44 +97,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=48) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (2048, 857)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (2048, 857)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (2048, 857)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, @@ -106,44 +111,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=48) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (2048, 1080)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (2048, 1080)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (2048, 1080)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, @@ -157,44 +125,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=24) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (2048, 1080)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (2048, 1080)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (2048, 1080)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, @@ -208,44 +139,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=24) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (2048, 857)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (2048, 857)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (2048, 857)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, @@ -259,44 +153,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=48) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (1998, 1080)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (1998, 1080)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (1998, 1080)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, @@ -310,44 +167,7 @@ class TestSuiteWrite(unittest.TestCase): j.write(data, cinema2k=24) codestream = j.get_codestream() - - # SIZ: Image and tile size - # Profile: "3" means cinema2K - self.assertEqual(codestream.segment[1].rsiz, 3) - # Reference grid size - self.assertEqual((codestream.segment[1].xsiz, - codestream.segment[1].ysiz), - (1998, 1080)) - # Reference grid offset - self.assertEqual((codestream.segment[1].xosiz, - codestream.segment[1].yosiz), (0, 0)) - # Tile size - self.assertEqual((codestream.segment[1].xtsiz, - codestream.segment[1].ytsiz), - (1998, 1080)) - # Tile offset - self.assertEqual((codestream.segment[1].xtosiz, - codestream.segment[1].ytosiz), - (0, 0)) - # bitdepth - self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) - # signed - self.assertEqual(codestream.segment[1].signed, - (False, False, False)) - # subsampling - self.assertEqual(list(zip(codestream.segment[1].xrsiz, - codestream.segment[1].yrsiz)), - [(1, 1)] * 3) - - # COD: Coding style default - self.assertFalse(codestream.segment[2].scod & 2) # no sop - self.assertFalse(codestream.segment[2].scod & 4) # no eph - self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) - self.assertEqual(codestream.segment[2].layers, 1) - self.assertEqual(codestream.segment[2].spcod[3], 1) # mct - self.assertEqual(codestream.segment[2].spcod[4], 5) # levels - self.assertEqual(tuple(codestream.segment[2].code_block_size), - (32, 32)) # cblksz + self.check_cinema2k_codestream(codestream, (1998, 1080)) @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, From d68b1aacb1f888adfef13cd2054421989607a21c Mon Sep 17 00:00:00 2001 From: John Evans Date: Fri, 7 Mar 2014 11:00:20 -0500 Subject: [PATCH 83/83] Cinema4K seems to be working. #139 --- CHANGES.txt | 2 +- docs/source/changelog.rst | 2 +- docs/source/detailed_installation.rst | 2 +- glymur/jp2k.py | 63 +++++++++++++++++++++------ glymur/test/test_opj_suite_write.py | 56 ++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 17 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index c906d29..44d6832 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,4 @@ -Mar 06, 2014 - Added Cinema2K write support. +Mar 06, 2014 - Added Cinema2K, Cinema4K write support. Changed constructor for ChannelDefinition box. Removed support for Python 2.6. Added write support for JP2 UUID, DataEntryURL, Palette and Component Mapping boxes, JPX Association, NumberList diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 320ff0f..7a96571 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -5,7 +5,7 @@ ChangeLog 0.6.0 (pending) =============== - * Added Cinema2K write support. + * Added Cinema2K, Cinema4K write support. * Added lxml requirement. * added set_printoptions, get_printoptions function * dropped support for Python 2.6, added support for Python 3.4 diff --git a/docs/source/detailed_installation.rst b/docs/source/detailed_installation.rst index ae0ce08..9111dbd 100644 --- a/docs/source/detailed_installation.rst +++ b/docs/source/detailed_installation.rst @@ -13,7 +13,7 @@ both read and write JPEG 2000 files, but you may wish to install version 2.0 or the 2.0+ version from OpenJPEG's development trunk for better performance. If you do that, you should compile it as a shared library (named *openjp2* instead of *openjpeg*) from the developmental source that you can retrieve -via subversion. As of this time of writing, svn revision r2366 works. +via subversion. As of this time of writing, svn revision r2369 works. You should also download the test data for the purpose of configuring and running OpenJPEG's test suite, check their instructions for all this. You should set the **OPJ_DATA_ROOT** environment variable for the purpose diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 00d3d24..e861053 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -163,22 +163,32 @@ class Jp2k(Jp2kBox): msg += "profile if the file type box brand is 'jp2 '." warnings.warn(msg) - def _set_cinema_params(self, cparams, fps): + def _set_cinema_params(self, cparams, cinema_mode, fps): """Populate compression parameters structure for cinema2K. Parameters ---------- + params : ctypes struct + Corresponds to compression parameters structure used by the + library. + cinema_mode : str + Either 'cinema2k' or 'cinema4k' fps : int Frames per second, should be either 24 or 48. """ - if fps == 24: - cparams.cp_cinema = CINEMA_MODE['cinema2k_24'] - elif fps == 48: - cparams.cp_cinema = CINEMA_MODE['cinema2k_48'] + if cinema_mode == 'cinema2k': + if fps == 24: + cparams.cp_cinema = CINEMA_MODE['cinema2k_24'] + elif fps == 48: + cparams.cp_cinema = CINEMA_MODE['cinema2k_48'] + else: + raise IOError('Cinema2K frame rate must be either 24 or 48.') + cparams.cp_rsiz = RSIZ['CINEMA2K'] else: - raise IOError('Cinema2K frame rate must be either 24 or 48.') + cparams.cp_cinema = CINEMA_MODE['cinema4k_24'] + cparams.cp_rsiz = RSIZ['CINEMA4K'] + - cparams.cp_rsiz = RSIZ['CINEMA2K'] # No tiling cparams.tile_size_on = opj2.FALSE cparams.cp_tdx = 1 @@ -216,9 +226,17 @@ class Jp2k(Jp2kBox): # TODO: warning or error cparams.tcp_numlayers = 1 - if cparams.numresolution > 6: #TODO only for cinema2k - # TODO: warning or error - cparams.numresolution = 6 + if cinema_mode == 'cinema2k': + if cparams.numresolution > 6: + # TODO: warning or error + cparams.numresolution = 6 + else: + if cparams.numresolution < 2: + # TODO: warning or error + cparams.numresolution = 1 + elif cparams.numresolution > 7: + cparams.numresolution = 7 + # precincts cparams.csty |= 0x01 @@ -230,8 +248,25 @@ class Jp2k(Jp2kBox): # Progression order shall be CPRL cparams.prog_order = PROGRESSION_ORDER['CPRL'] - # progression order changes not allowed for 2K # TODO not for 4K: - cparams.numpocs = 0 + # progression order changes not allowed for 2K + if cinema_mode == 'cinema2k': + cparams.numpocs = 0 + else: + cparams.poc[0].tile = 1 + cparams.poc[0].resno0 = 0 + cparams.poc[0].compno0 = 0 + cparams.poc[0].layno1 = 1 + cparams.poc[0].resno1 = cparams.numresolution - 1 + cparams.poc[0].compno1 = 3 + cparams.poc[0].prg1 = PROGRESSION_ORDER['CPRL'] + cparams.poc[1].tile = 1 + cparams.poc[1].resno0 = 0 + cparams.poc[1].compno0 = 0 + cparams.poc[1].layno1 = 1 + cparams.poc[1].resno1 = cparams.numresolution + cparams.poc[1].compno1 = 3 + cparams.poc[1].prg1 = PROGRESSION_ORDER['CPRL'] + cparams.numpocs = 2 def _populate_cparams(self, **kwargs): """Populate compression parameters structure from input arguments. @@ -300,11 +335,11 @@ class Jp2k(Jp2kBox): cparams.cp_disto_alloc = 1 if 'cinema2k' in kwargs: - self._set_cinema_params(cparams, kwargs['cinema2k']) + self._set_cinema_params(cparams, 'cinema2k', kwargs['cinema2k']) return cparams if 'cinema4k' in kwargs: - self._set_cinema_params(cparams, kwargs['cinema2k']) + self._set_cinema_params(cparams, 'cinema4k', kwargs['cinema4k']) return cparams if 'cbsize' in kwargs: diff --git a/glymur/test/test_opj_suite_write.py b/glymur/test/test_opj_suite_write.py index 87a6fc4..0e7f2d3 100644 --- a/glymur/test/test_opj_suite_write.py +++ b/glymur/test/test_opj_suite_write.py @@ -44,6 +44,48 @@ class TestSuiteWrite(unittest.TestCase): def tearDown(self): pass + def check_cinema4k_codestream(self, codestream, image_size): + """Common out for cinema2k tests.""" + # SIZ: Image and tile size + # Profile: "3" means cinema2K + self.assertEqual(codestream.segment[1].rsiz, 4) + # Reference grid size + self.assertEqual((codestream.segment[1].xsiz, + codestream.segment[1].ysiz), + image_size) + # Reference grid offset + self.assertEqual((codestream.segment[1].xosiz, + codestream.segment[1].yosiz), (0, 0)) + # Tile size + self.assertEqual((codestream.segment[1].xtsiz, + codestream.segment[1].ytsiz), + image_size) + # Tile offset + self.assertEqual((codestream.segment[1].xtosiz, + codestream.segment[1].ytosiz), + (0, 0)) + # bitdepth + self.assertEqual(codestream.segment[1].bitdepth, (12, 12, 12)) + # signed + self.assertEqual(codestream.segment[1].signed, + (False, False, False)) + # subsampling + self.assertEqual(list(zip(codestream.segment[1].xrsiz, + codestream.segment[1].yrsiz)), + [(1, 1)] * 3) + + # COD: Coding style default + self.assertFalse(codestream.segment[2].scod & 2) # no sop + self.assertFalse(codestream.segment[2].scod & 4) # no eph + self.assertEqual(codestream.segment[2].spcod[0], glymur.core.CPRL) + self.assertEqual(codestream.segment[2].layers, 1) + self.assertEqual(codestream.segment[2].spcod[3], 1) # mct + self.assertEqual(codestream.segment[2].spcod[4], 5) # levels + self.assertEqual(tuple(codestream.segment[2].code_block_size), + (32, 32)) # cblksz + + + def check_cinema2k_codestream(self, codestream, image_size): """Common out for cinema2k tests.""" # SIZ: Image and tile size @@ -86,6 +128,20 @@ class TestSuiteWrite(unittest.TestCase): + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, + "Cannot read input image without scikit-image/freeimage") + def test_NR_ENC_ElephantDream_4K_tif_21_encode(self): + relfile = 'input/nonregression/ElephantDream_4K.tif' + infile = opj_data_file(relfile) + data = skimage.io.imread(infile) + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j.write(data, cinema4k=True) + + codestream = j.get_codestream() + self.check_cinema4k_codestream(codestream, (4096, 2160)) + + @unittest.skipIf(not _HAS_SKIMAGE_FREEIMAGE_SUPPORT, "Cannot read input image without scikit-image/freeimage") def test_NR_ENC_X_5_2K_24_235_CBR_STEM24_000_tif_19_encode(self):