Merge branch 'issue96' into devel

glymur/codestream.py

@@ -556,8 +556,8 @@ class Codestream(object):
             mantissa_exponent_buffer_length = length - 4
         cqcc, sqcc = struct.unpack(fmt, read_buffer)
         if cqcc >= self._csiz:
-            msg = "Invalid component number (%d), "
+            msg = "Invalid component number ({0}), "
-            msg += "number of components is only %d."
+            msg += "number of components is only {1}."
             msg = msg.format(cqcc, self._csiz)
             warnings.warn(msg)
 

glymur/jp2box.py

@@ -1792,14 +1792,14 @@ class XMLBox(Jp2kBox):
         text = read_buffer.decode('utf-8')
 
         # Strip out any trailing nulls, as they can foul up XML parsing.
-        text = text.rstrip('\0')
+        text = text.rstrip(chr(0))
 
         try:
             elt = ET.fromstring(text)
             xml = ET.ElementTree(elt)
         except ParseError as parse_error:
             msg = 'A problem was encountered while parsing an XML box:'
-            msg += '\n\n\t"{0}"\n\nNo XML was retrieved.'  
+            msg += '\n\n\t"{0}"\n\nNo XML was retrieved.'
             msg = msg.format(str(parse_error))
             warnings.warn(msg, UserWarning)
             xml = None

glymur/jp2dump.py

@@ -1,6 +1,7 @@
 """
 Entry point for jp2dump script.
 """
+import warnings
 
 from .jp2k import Jp2k
 
@@ -15,8 +16,21 @@ def jp2dump(filename, codestream=False):
     codestream : optional, logical scalar
         Whether or not to dump codestream contents.
     """
-    j = Jp2k(filename)
+    with warnings.catch_warnings(record=True) as wctx:
-    if codestream:
+
-        print(j.get_codestream(header_only=False))
+        # JP2 metadata can be extensive, so don't print any warnings until we
-    else:
+        # are done with the metadata.
-        print(j)
+        j = Jp2k(filename)
+        if codestream:
+            print(j.get_codestream(header_only=False))
+        else:
+            print(j)
+
+        # Re-emit any warnings that may have been suppressed.
+        if len(wctx) > 0:
+            print("\n")
+        for warning in wctx:
+            print("{0}:{1}: {2}: {3}".format(warning.filename,
+                                             warning.lineno,
+                                             warning.category.__name__,
+                                             warning.message))

glymur/jp2k.py

@@ -966,8 +966,10 @@ class Jp2k(Jp2kBox):
             dparam.nb_tile_to_decode = 1
 
         with ExitStack() as stack:
-            if hasattr(_opj2.OPENJP2, 'opj_stream_create_default_file_stream_v3'):
+            if hasattr(_opj2.OPENJP2,
-                stream = _opj2.stream_create_default_file_stream_v3(self.filename,
+                       'opj_stream_create_default_file_stream_v3'):
+                filename = self.filename
+                stream = _opj2.stream_create_default_file_stream_v3(filename,
                                                                     True)
                 stack.callback(_opj2.stream_destroy_v3, stream)
             else:

glymur/test/__init__.py

@@ -0,0 +1,3 @@
+"""
+Test suite for glymur high-level functionality.
+"""

glymur/test/fixtures.py

@@ -1,5 +1,9 @@
+"""
+Test fixtures common to more than one test point.
+"""
 import re
 import sys
+import warnings
 
 import numpy as np
 
@@ -20,6 +24,28 @@ if glymur.lib.openjp2.OPENJP2 is not None:
         OPENJP2_IS_V2_OFFICIAL = True
 
 
+NO_READ_BACKEND_MSG = "Matplotlib with the PIL backend must be available in "
+NO_READ_BACKEND_MSG += "order to run the tests in this suite."
+
+try:
+    from matplotlib.pyplot import imread
+    NO_READ_BACKEND = False
+except ImportError:
+    NO_READ_BACKEND = True
+
+
+def read_image(infile):
+    """Read image using matplotlib backend.
+    
+    Hopefully PIL(low) is installed as matplotlib's backend.  It issues 
+    warnings which we do not care about, so suppress them.
+    """
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        data = imread(infile)
+    return data
+
+
 def mse(amat, bmat):
     """Mean Square Error"""
     diff = amat.astype(np.double) - bmat.astype(np.double)
@@ -36,28 +62,10 @@ def peak_tolerance(amat, bmat):
 
 def read_pgx(pgx_file):
     """Helper function for reading the PGX comparison files.
-
-    Open the file in ascii mode and read the header line.
-    Will look something like
-
-    PG ML + 8 128 128
-    PG%[ \t]%c%c%[ \t+-]%d%[ \t]%d%[ \t]%d"
     """
-    header = ''
+    header, pos = read_pgx_header(pgx_file)
-    with open(pgx_file, 'rb') as fptr:
-        while True:
-            char = fptr.read(1)
-            if char[0] == 10 or char == '\n':
-                pos = fptr.tell()
-                break
-            else:
-                if sys.hexversion < 0x03000000:
-                    header += char
-                else:
-                    header += chr(char[0])
 
-    header = header.rstrip()
+    tokens = re.split(r'\s', header)
-    tokens = re.split('\s', header)
 
     if (tokens[1][0] == 'M') and (sys.byteorder == 'little'):
         swapbytes = True
@@ -81,6 +89,28 @@ def read_pgx(pgx_file):
         nrows = int(tokens[4])
         ncols = int(tokens[3])
 
+    dtype = determine_pgx_datatype(signed, bitdepth)
+
+    shape = [nrows, ncols]
+
+    # Reopen the file in binary mode and seek to the start of the binary
+    # data
+    with open(pgx_file, 'rb') as fptr:
+        fptr.seek(pos)
+        data = np.fromfile(file=fptr, dtype=dtype).reshape(shape)
+
+    return(data.byteswap(swapbytes))
+
+def determine_pgx_datatype(signed, bitdepth):
+    """Determine the datatype of the PGX file.
+
+    Parameters
+    ----------
+    signed : bool
+        True if the datatype is signed, false otherwise
+    bitdepth : int
+        How many bits are used to make up an image plane.  Should be 8 or 16.
+    """
     if signed:
         if bitdepth <= 8:
             dtype = np.int8
@@ -96,12 +126,28 @@ def read_pgx(pgx_file):
         else:
             raise RuntimeError("unhandled bitdepth")
 
-    shape = [nrows, ncols]
+    return dtype
 
-    # Reopen the file in binary mode and seek to the start of the binary
+def read_pgx_header(pgx_file):
-    # data
+    """Open the file in ascii mode (not really) and read the header line.
+    Will look something like
+
+    PG ML + 8 128 128
+    PG%[ \t]%c%c%[ \t+-]%d%[ \t]%d%[ \t]%d"
+    """
+    header = ''
     with open(pgx_file, 'rb') as fptr:
-        fptr.seek(pos)
+        while True:
-        data = np.fromfile(file=fptr, dtype=dtype).reshape(shape)
+            char = fptr.read(1)
+            if char[0] == 10 or char == '\n':
+                pos = fptr.tell()
+                break
+            else:
+                if sys.hexversion < 0x03000000:
+                    header += char
+                else:
+                    header += chr(char[0])
+
+    header = header.rstrip()
+    return header, pos
 
-    return(data.byteswap(swapbytes))

glymur/test/test_callbacks.py

@@ -88,7 +88,7 @@ class TestCallbacks15(unittest.TestCase):
             with patch('sys.stdout', new=StringIO()) as fake_out:
                 d = j.read(rlevel=1, verbose=True)
                 actual = fake_out.getvalue().strip()
-    
+
             regex = re.compile(r"""\[INFO\]\stile\s1\sof\s1\s+
                                    \[INFO\]\s-\stiers-1\stook\s
                                            [0-9]+\.[0-9]+\ss\s+

glymur/test/test_jp2k.py

@@ -7,6 +7,7 @@ import struct
 import sys
 import tempfile
 import uuid
+from xml.etree import cElementTree as ET
 
 if sys.hexversion < 0x02070000:
     import unittest2 as unittest
@@ -65,7 +66,7 @@ class TestConfig(unittest.TestCase):
         """Don't have either openjp2 or openjpeg libraries?  Must error out.
         """
         with patch('glymur.lib.openjp2.OPENJP2', new=None):
-            with  patch('glymur.lib.openjpeg.OPENJPEG', new=None):
+            with patch('glymur.lib.openjpeg.OPENJPEG', new=None):
                 with self.assertRaises(glymur.jp2k.LibraryNotFoundError):
                     d = glymur.Jp2k(self.jp2file).read()
 
@@ -73,7 +74,7 @@ class TestConfig(unittest.TestCase):
         """Don't have openjp2 library?  Must error out.
         """
         with patch('glymur.lib.openjp2.OPENJP2', new=None):
-            with  patch('glymur.lib.openjpeg.OPENJPEG', new=None):
+            with patch('glymur.lib.openjpeg.OPENJPEG', new=None):
                 with self.assertRaises(glymur.jp2k.LibraryNotFoundError):
                     d = glymur.Jp2k(self.jp2file).read_bands()
 
@@ -83,7 +84,7 @@ class TestConfig(unittest.TestCase):
         """
         data = glymur.Jp2k(self.j2kfile).read()
         with patch('glymur.lib.openjp2.OPENJP2', new=None):
-            with  patch('glymur.lib.openjpeg.OPENJPEG', new=None):
+            with patch('glymur.lib.openjpeg.OPENJPEG', new=None):
                 with self.assertRaises(glymur.jp2k.LibraryNotFoundError):
                     with tempfile.NamedTemporaryFile(suffix='.jp2') as tfile:
                         ofile = Jp2k(tfile.name, 'wb')
@@ -623,6 +624,8 @@ class TestJp2k(unittest.TestCase):
             self.assertEqual(jp2k.box[3].box_id, 'xml ')
             self.assertEqual(jp2k.box[3].offset, 77)
             self.assertEqual(jp2k.box[3].length, 36)
+            self.assertEqual(ET.tostring(jp2k.box[3].xml.getroot()),
+                             b'<test>this is a test</test>')
 
     @unittest.skipIf(os.name == "nt", "NamedTemporaryFile issue on windows")
     def test_asoc_label_box(self):

glymur/test/test_opj_suite.py

@@ -66,11 +66,10 @@ class TestSuite(unittest.TestCase):
 
     def test_ETS_C0P0_p0_02_j2k(self):
         jfile = os.path.join(data_root, 'input/conformance/p0_02.j2k')
+        jp2k = Jp2k(jfile)
         with warnings.catch_warnings():
-            # There's a 0xff30 marker segment.  Not illegal, but we don't
+            # Invalid marker ID.
-            # really know what to do with it.  Just ignore.
             warnings.simplefilter("ignore")
-            jp2k = Jp2k(jfile)
             jpdata = jp2k.read(rlevel=0)
 
         pgxfile = os.path.join(data_root, 'baseline/conformance/c0p0_02.pgx')
@@ -385,11 +384,7 @@ class TestSuite(unittest.TestCase):
 
     def test_ETS_C1P0_p0_02_j2k(self):
         jfile = os.path.join(data_root, 'input/conformance/p0_02.j2k')
-        with warnings.catch_warnings():
+        jp2k = Jp2k(jfile)
-            # There's a 0xff30 marker segment.  Not illegal, but we don't
-            # really know what to do with it.  Just ignore.
-            warnings.simplefilter("ignore")
-            jp2k = Jp2k(jfile)
         jpdata = jp2k.read(rlevel=0)
 
         pgxfile = os.path.join(data_root, 'baseline/conformance/c1p0_02_0.pgx')
@@ -830,7 +825,7 @@ class TestSuite(unittest.TestCase):
         jfile = os.path.join(data_root, 'input/nonregression/broken2.jp2')
         with self.assertRaises(IOError):
             with warnings.catch_warnings():
-                # Library warning, invalid number of subbands.
+                # Invalid marker ID.
                 warnings.simplefilter("ignore")
                 data = Jp2k(jfile).read()
         self.assertTrue(True)
@@ -852,7 +847,7 @@ class TestSuite(unittest.TestCase):
                              'input/nonregression/broken4.jp2')
         with self.assertRaises(IOError):
             with warnings.catch_warnings():
-                # Library warning, invalid number of subbands.
+                # invalid number of subbands, bad marker ID
                 warnings.simplefilter("ignore")
                 data = Jp2k(jfile).read()
         self.assertTrue(True)
@@ -1053,6 +1048,7 @@ class TestSuite(unittest.TestCase):
         f = 'input/nonregression/gdal_fuzzer_unchecked_numresolutions.jp2'
         jfile = os.path.join(data_root, f)
         with warnings.catch_warnings():
+            # Invalid number of resolutions.
             warnings.simplefilter("ignore")
             j = Jp2k(jfile)
             with self.assertRaises(IOError):
@@ -1066,6 +1062,7 @@ class TestSuite(unittest.TestCase):
                'gdal_fuzzer_assert_in_opj_j2k_read_SQcd_SQcc.patch.jp2')
         jfile = os.path.join(data_root, '/'.join(lst))
         with warnings.catch_warnings():
+            # Invalid component number.
             warnings.simplefilter("ignore")
             j = Jp2k(jfile)
             with self.assertRaises(IOError):
@@ -1077,6 +1074,7 @@ class TestSuite(unittest.TestCase):
         relpath = 'input/nonregression/gdal_fuzzer_check_number_of_tiles.jp2'
         jfile = os.path.join(data_root, relpath)
         with warnings.catch_warnings():
+            # Invalid number of tiles.
             warnings.simplefilter("ignore")
             j = Jp2k(jfile)
             with self.assertRaises(IOError):
@@ -1088,6 +1086,7 @@ class TestSuite(unittest.TestCase):
         relpath = 'input/nonregression/gdal_fuzzer_check_comp_dx_dy.jp2'
         jfile = os.path.join(data_root, relpath)
         with warnings.catch_warnings():
+            # Invalid subsampling value
             warnings.simplefilter("ignore")
             with self.assertRaises(IOError):
                 j = Jp2k(jfile).read()
@@ -1220,36 +1219,28 @@ class TestSuite(unittest.TestCase):
     def test_NR_DEC_p1_04_j2k_57_decode(self):
         jfile = os.path.join(data_root, 'input/conformance/p1_04.j2k')
         jp2k = Jp2k(jfile)
-        with warnings.catch_warnings():
+        tdata = jp2k.read(tile=63)  # last tile
-            warnings.simplefilter("ignore")
-            tdata = jp2k.read(tile=63)  # last tile
         odata = jp2k.read()
         np.testing.assert_array_equal(tdata, odata[896:1024, 896:1024])
 
     def test_NR_DEC_p1_04_j2k_58_decode(self):
         jfile = os.path.join(data_root, 'input/conformance/p1_04.j2k')
         jp2k = Jp2k(jfile)
-        with warnings.catch_warnings():
+        tdata = jp2k.read(tile=63, rlevel=2)  # last tile
-            warnings.simplefilter("ignore")
-            tdata = jp2k.read(tile=63, rlevel=2)  # last tile
         odata = jp2k.read(rlevel=2)
         np.testing.assert_array_equal(tdata, odata[224:256, 224:256])
 
     def test_NR_DEC_p1_04_j2k_59_decode(self):
         jfile = os.path.join(data_root, 'input/conformance/p1_04.j2k')
         jp2k = Jp2k(jfile)
-        with warnings.catch_warnings():
+        tdata = jp2k.read(tile=12)  # 2nd row, 5th column
-            warnings.simplefilter("ignore")
-            tdata = jp2k.read(tile=12)  # 2nd row, 5th column
         odata = jp2k.read()
         np.testing.assert_array_equal(tdata, odata[128:256, 512:640])
 
     def test_NR_DEC_p1_04_j2k_60_decode(self):
         jfile = os.path.join(data_root, 'input/conformance/p1_04.j2k')
         jp2k = Jp2k(jfile)
-        with warnings.catch_warnings():
+        tdata = jp2k.read(tile=12, rlevel=1)  # 2nd row, 5th column
-            warnings.simplefilter("ignore")
-            tdata = jp2k.read(tile=12, rlevel=1)  # 2nd row, 5th column
         odata = jp2k.read(rlevel=1)
         np.testing.assert_array_equal(tdata, odata[64:128, 256:320])
 
@@ -1366,9 +1357,7 @@ class TestSuite(unittest.TestCase):
         jfile = os.path.join(data_root, 'input/conformance/p1_06.j2k')
         jp2k = Jp2k(jfile)
         fulldata = jp2k.read()
-        with warnings.catch_warnings():
+        tiledata = jp2k.read(tile=0)
-            warnings.simplefilter("ignore")
-            tiledata = jp2k.read(tile=0)
         np.testing.assert_array_equal(tiledata, fulldata[0:3, 0:3])
 
     @unittest.skip("fprintf stderr output in r2343.")
@@ -1376,9 +1365,7 @@ class TestSuite(unittest.TestCase):
         jfile = os.path.join(data_root, 'input/conformance/p1_06.j2k')
         jp2k = Jp2k(jfile)
         fulldata = jp2k.read()
-        with warnings.catch_warnings():
+        tiledata = jp2k.read(tile=5)
-            warnings.simplefilter("ignore")
-            tiledata = jp2k.read(tile=5)
         np.testing.assert_array_equal(tiledata, fulldata[3:6, 3:6])
 
     @unittest.skip("fprintf stderr output in r2343.")
@@ -3809,7 +3796,7 @@ class TestSuiteDump(unittest.TestCase):
                           32, 32, 131, 2002, 2002, 1888])
         self.assertEqual(c.segment[3]._exponent,
                          [17, 17, 17, 17, 16, 16, 16, 15, 15, 15, 14, 14,
-                         14, 13, 13, 13, 11, 11, 11, 11, 11, 11])
+                          14, 13, 13, 13, 11, 11, 11, 11, 11, 11])
 
         # COM: comment
         # Registration
@@ -4596,10 +4583,10 @@ class TestSuiteDump(unittest.TestCase):
         self.assertEqual(c.segment[7].sqcc & 0x1f, 2)  # none
         self.assertEqual(c.segment[7]._mantissa,
                          [1824, 1776, 1776, 1728, 1792, 1792, 1760, 1872,
-                         1872, 1896, 5, 5, 71, 2003, 2003, 1890])
+                          1872, 1896, 5, 5, 71, 2003, 2003, 1890])
         self.assertEqual(c.segment[7]._exponent,
                          [18, 18, 18, 18, 17, 17, 17, 16, 16, 16, 14, 14,
-                         14, 14, 14, 14])
+                          14, 14, 14, 14])
 
         # COM: comment
         # Registration
@@ -5634,7 +5621,7 @@ class TestSuiteDump(unittest.TestCase):
         self.assertEqual(c.segment[3]._mantissa, [0] * 16)
         self.assertEqual(c.segment[3]._exponent,
                          [18, 19, 19, 20, 19, 19, 20, 19, 19, 20, 19, 19, 20,
-                         19, 19, 20])
+                          19, 19, 20])
 
         # COM: comment
         # Registration

glymur/test/test_opj_suite_neg.py

@@ -12,22 +12,12 @@ if sys.hexversion < 0x02070000:
 else:
     import unittest
 
-import warnings
-
 import numpy as np
 import pkg_resources
 
 from glymur.lib import openjp2 as opj2
 
-msg = "Matplotlib with the PIL backend must be available in order to run the "
+from .fixtures import read_image, NO_READ_BACKEND, NO_READ_BACKEND_MSG
-msg += "tests in this suite."
-no_read_backend_msg = msg
-try:
-    from PIL import Image
-    from matplotlib.pyplot import imread
-    no_read_backend = False
-except:
-    no_read_backend = True
 
 from glymur import Jp2k
 import glymur
@@ -40,17 +30,9 @@ except:
     raise
 
 
-def read_image(infile):
-    # PIL issues warnings which we do not care about, so suppress them.
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        data = imread(infile)
-    return data
-
-
 @unittest.skipIf(glymur.lib.openjp2.OPENJP2 is None,
                  "Missing openjp2 library.")
-@unittest.skipIf(no_read_backend, no_read_backend_msg)
+@unittest.skipIf(NO_READ_BACKEND, NO_READ_BACKEND_MSG)
 @unittest.skipIf(data_root is None,
                  "OPJ_DATA_ROOT environment variable not set")
 class TestSuiteNegative(unittest.TestCase):

glymur/test/test_opj_suite_write.py

@@ -13,21 +13,11 @@ if sys.hexversion < 0x02070000:
 else:
     import unittest
 
-import warnings
-
 import numpy as np
 
 from glymur.lib import openjp2 as opj2
 
-msg = "Matplotlib with the PIL backend must be available in order to run the "
+from .fixtures import read_image, NO_READ_BACKEND, NO_READ_BACKEND_MSG
-msg += "tests in this suite."
-no_read_backend_msg = msg
-try:
-    from PIL import Image
-    from matplotlib.pyplot import imread
-    no_read_backend = False
-except:
-    no_read_backend = True
 
 from glymur import Jp2k
 import glymur
@@ -40,18 +30,10 @@ except:
     raise
 
 
-def read_image(infile):
-    # PIL issues warnings which we do not care about, so suppress them.
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        data = imread(infile)
-    return data
-
-
 @unittest.skipIf(os.name == "nt", "no write support on windows, period")
 @unittest.skipIf(glymur.lib.openjp2.OPENJP2 is None,
                  "Missing openjp2 library.")
-@unittest.skipIf(no_read_backend, no_read_backend_msg)
+@unittest.skipIf(NO_READ_BACKEND, NO_READ_BACKEND_MSG)
 @unittest.skipIf(data_root is None,
                  "OPJ_DATA_ROOT environment variable not set")
 class TestSuiteWrite(unittest.TestCase):

glymur/test/test_printing.py

@@ -1,6 +1,7 @@
 #pylint:  disable-all
 import os
 import pkg_resources
+import re
 import struct
 import sys
 import tempfile
@@ -1026,5 +1027,6 @@ class TestPrinting(unittest.TestCase):
 
         self.assertEqual(actual, expected)
 
+
 if __name__ == "__main__":
     unittest.main()