From 1fd0527c3c3bbf1f995a240851221b32c2c4eb13 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 4 Oct 2014 19:43:40 -0400 Subject: [PATCH 1/8] implemented most common ellipsis use cases --- glymur/jp2k.py | 24 +++++++++++++++++++----- glymur/test/test_jp2k.py | 25 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 87028e1..ea8e955 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -763,10 +763,10 @@ class Jp2k(Jp2kBox): """ Slicing protocol. """ - if isinstance(index, slice) and ( - index.start == None and + if ((isinstance(index, slice) and + (index.start == None and index.stop == None and - index.step == None): + index.step == None)) or (index == ...)): # Case of jp2[:] = data, i.e. write the entire image. # # Should have a slice object where start = stop = step = None @@ -787,8 +787,8 @@ class Jp2k(Jp2kBox): area = (row, 0, row + 1, codestream.segment[1].xsiz) return self.read(area=area).squeeze() - if isinstance(pargs, slice): - # Case of jp2[:], i.e. retrieve the entire image. + if isinstance(pargs, slice) or pargs is ...: + # Case of jp2[:] or jp2[...], i.e. retrieve the entire image. # # Should have a slice object where start = stop = step = None return self.read() @@ -806,6 +806,20 @@ class Jp2k(Jp2kBox): elif len(pargs) == 3: return pixel[pargs[2]] + if isinstance(pargs, tuple) and any(x is ... for x in pargs): + + nrows = codestream.segment[1].ysiz + ncols = codestream.segment[1].xsiz + nbands = codestream.segment[1].Csiz + # Reformulate without the ellipsis. + if pargs[0] is ...: + newindex = (slice(0, nrows), slice(0, ncols), pargs[1]) + elif pargs[1] is ...: + # Assume we have something like (r,...) where r is a scalar. + newindex = pargs[0] + + return self.__getitem__(newindex) + # Assuming pargs is a tuple of slices from now on. rows = pargs[0] cols = pargs[1] diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index 2868e84..ef86744 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -71,6 +71,16 @@ class SliceProtocolBase(unittest.TestCase): @unittest.skipIf(os.name == "nt", "NamedTemporaryFile issue on windows") class TestSliceProtocolBaseWrite(SliceProtocolBase): + def test_write_ellipsis(self): + expected = self.j2k_data + + with tempfile.NamedTemporaryFile(suffix='.j2k') as tfile: + j = Jp2k(tfile.name, 'wb') + j[...] = self.j2k_data + actual = j.read() + + np.testing.assert_array_equal(actual, expected) + def test_basic_write(self): expected = self.j2k_data @@ -236,6 +246,21 @@ class TestSliceProtocolRead(SliceProtocolBase): expected = self.jp2.read(area=(0, 0, 202, 202), rlevel=1) np.testing.assert_array_equal(actual, expected) + def test_ellipsis_full_read(self): + actual = self.j2k[...] + expected = self.j2k_data + np.testing.assert_array_equal(actual, expected) + + def test_ellipsis_band_select(self): + actual = self.j2k[..., 0] + expected = self.j2k_data[..., 0] + np.testing.assert_array_equal(actual, expected) + + def test_ellipsis_row_select(self): + actual = self.j2k[0, ...] + expected = self.j2k_data[0, ...] + np.testing.assert_array_equal(actual, expected) + def test_slice_protocol_2d_reduce_resolution(self): d = self.j2k[:] self.assertEqual(d.shape, (800, 480, 3)) From 50cccf89be88a90f1def42a22f1b7dcf7c955de7 Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 4 Oct 2014 22:16:09 -0400 Subject: [PATCH 2/8] additional ellipsis cases --- glymur/jp2k.py | 30 ++++++++++++++++++++++++++---- glymur/test/test_jp2k.py | 10 ++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index ea8e955..751d378 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -18,6 +18,7 @@ else: from collections import Counter import ctypes +import itertools import math import os import re @@ -807,19 +808,40 @@ class Jp2k(Jp2kBox): return pixel[pargs[2]] if isinstance(pargs, tuple) and any(x is ... for x in pargs): - nrows = codestream.segment[1].ysiz ncols = codestream.segment[1].xsiz nbands = codestream.segment[1].Csiz # Reformulate without the ellipsis. if pargs[0] is ...: - newindex = (slice(0, nrows), slice(0, ncols), pargs[1]) + if len(pargs) == 2: + newindex = (slice(0, nrows), slice(0, ncols), pargs[1]) + else: + newindex = (slice(0, nrows), pargs[1], pargs[2]) elif pargs[1] is ...: - # Assume we have something like (r,...) where r is a scalar. - newindex = pargs[0] + if len(pargs) == 2: + newindex = (pargs[0], slice(0, ncols), slice(0, nbands)) + else: + newindex = (pargs[0], slice(0, ncols), pargs[2]) + else: + newindex = (pargs[0], pargs[1], slice(0, nbands)) return self.__getitem__(newindex) + if isinstance(pargs, tuple) and not all(isinstance(x, slice) for x in pargs): + # Search out any remaining non-slices and turn them into slices. + lst = list(pargs) + predicate = lambda x: not isinstance(x[1], int) + g = itertools.filterfalse(predicate, enumerate(pargs)) + idx = list(g)[0][0] + lst[idx] = slice(pargs[idx], pargs[idx] + 1) + newindex = tuple(lst) + + data = self.__getitem__(newindex) + + # Reduce dimensionality in the scalar dimension. + return np.squeeze(data, axis=idx) + + # Assuming pargs is a tuple of slices from now on. rows = pargs[0] cols = pargs[1] diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index ef86744..77d796d 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -261,6 +261,16 @@ class TestSliceProtocolRead(SliceProtocolBase): expected = self.j2k_data[0, ...] np.testing.assert_array_equal(actual, expected) + def test_two_ellipsis_band_select(self): + actual = self.j2k[..., ..., 1] + expected = self.j2k_data[:, :, 1] + np.testing.assert_array_equal(actual, expected) + + def test_two_ellipsis_row_select(self): + actual = self.j2k[1, ..., ...] + expected = self.j2k_data[1, :, :] + np.testing.assert_array_equal(actual, expected) + def test_slice_protocol_2d_reduce_resolution(self): d = self.j2k[:] self.assertEqual(d.shape, (800, 480, 3)) From 6338eb4e22d50bf880eb47e07030e6784e507fee Mon Sep 17 00:00:00 2001 From: jevans Date: Sat, 4 Oct 2014 23:10:04 -0400 Subject: [PATCH 3/8] updated for python2, some refactoring --- glymur/jp2k.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 751d378..00911d2 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -13,12 +13,13 @@ import sys # pylint: disable=E0611 if sys.hexversion >= 0x03030000: from contextlib import ExitStack + from itertools import filterfalse else: from contextlib2 import ExitStack + from itertools import ifilterfalse as filterfalse from collections import Counter import ctypes -import itertools import math import os import re @@ -29,9 +30,6 @@ import warnings import numpy as np from .codestream import Codestream -from .core import SRGB, GREYSCALE -from .core import PROGRESSION_ORDER -from .core import ENUMERATED_COLORSPACE, RESTRICTED_ICC_PROFILE from . import core from .jp2box import Jp2kBox from .jp2box import JPEG2000SignatureBox, FileTypeBox, JP2HeaderBox @@ -149,8 +147,8 @@ class Jp2k(Jp2kBox): jp2h = [box for box in self.box if box.box_id == 'jp2h'][0] colrs = [box for box in jp2h.box if box.box_id == 'colr'] for colr in colrs: - if colr.method not in (ENUMERATED_COLORSPACE, - RESTRICTED_ICC_PROFILE): + if colr.method not in (core.ENUMERATED_COLORSPACE, + core.RESTRICTED_ICC_PROFILE): msg = "Color Specification box method must specify either " msg += "an enumerated colorspace or a restricted ICC " msg += "profile if the file type box brand is 'jp2 '." @@ -311,7 +309,7 @@ class Jp2k(Jp2kBox): if 'prog' in kwargs: prog = kwargs['prog'].upper() - cparams.prog_order = PROGRESSION_ORDER[prog] + cparams.prog_order = core.PROGRESSION_ORDER[prog] if 'psnr' in kwargs: cparams.tcp_numlayers = len(kwargs['psnr']) @@ -743,11 +741,11 @@ class Jp2k(Jp2kBox): width = codestream.segment[1].xsiz num_components = len(codestream.segment[1].xrsiz) if num_components < 3: - colorspace = GREYSCALE + colorspace = core.GREYSCALE else: if len(self.box) == 0: # Best guess is SRGB - colorspace = SRGB + colorspace = core.SRGB else: # Take whatever the first jp2 header / color specification # says. @@ -767,7 +765,7 @@ class Jp2k(Jp2kBox): if ((isinstance(index, slice) and (index.start == None and index.stop == None and - index.step == None)) or (index == ...)): + index.step == None)) or (index is Ellipsis)): # Case of jp2[:] = data, i.e. write the entire image. # # Should have a slice object where start = stop = step = None @@ -788,7 +786,8 @@ class Jp2k(Jp2kBox): area = (row, 0, row + 1, codestream.segment[1].xsiz) return self.read(area=area).squeeze() - if isinstance(pargs, slice) or pargs is ...: + #if isinstance(pargs, slice) or pargs is ...: + if isinstance(pargs, slice) or pargs is Ellipsis: # Case of jp2[:] or jp2[...], i.e. retrieve the entire image. # # Should have a slice object where start = stop = step = None @@ -807,17 +806,17 @@ class Jp2k(Jp2kBox): elif len(pargs) == 3: return pixel[pargs[2]] - if isinstance(pargs, tuple) and any(x is ... for x in pargs): + if isinstance(pargs, tuple) and any(x is Ellipsis for x in pargs): nrows = codestream.segment[1].ysiz ncols = codestream.segment[1].xsiz nbands = codestream.segment[1].Csiz # Reformulate without the ellipsis. - if pargs[0] is ...: + if pargs[0] is Ellipsis: if len(pargs) == 2: newindex = (slice(0, nrows), slice(0, ncols), pargs[1]) else: newindex = (slice(0, nrows), pargs[1], pargs[2]) - elif pargs[1] is ...: + elif pargs[1] is Ellipsis: if len(pargs) == 2: newindex = (pargs[0], slice(0, ncols), slice(0, nbands)) else: @@ -831,7 +830,7 @@ class Jp2k(Jp2kBox): # Search out any remaining non-slices and turn them into slices. lst = list(pargs) predicate = lambda x: not isinstance(x[1], int) - g = itertools.filterfalse(predicate, enumerate(pargs)) + g = filterfalse(predicate, enumerate(pargs)) idx = list(g)[0][0] lst[idx] = slice(pargs[idx], pargs[idx] + 1) newindex = tuple(lst) @@ -1511,14 +1510,14 @@ def _validate_channel_definition(jp2h, colr): raise IOError(msg) elif len(cdef_lst) == 1: cdef = jp2h.box[cdef_lst[0]] - if colr.colorspace == SRGB: + if colr.colorspace == core.SRGB: if any([chan + 1 not in cdef.association or cdef.channel_type[chan] != 0 for chan in [0, 1, 2]]): msg = "All color channels must be defined in the " msg += "channel definition box." raise IOError(msg) - elif colr.colorspace == GREYSCALE: + elif colr.colorspace == core.GREYSCALE: if 0 not in cdef.channel_type: msg = "All color channels must be defined in the " msg += "channel definition box." From 14e428b1ae4f5977cc84002bf755740120e64cbf Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 5 Oct 2014 20:23:19 -0400 Subject: [PATCH 4/8] removed unneeded block of code Checking for ints in the argument tuple was being done twice. --- glymur/jp2k.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 00911d2..7b9954e 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -793,19 +793,6 @@ class Jp2k(Jp2kBox): # Should have a slice object where start = stop = step = None return self.read() - if isinstance(pargs, tuple) and all(isinstance(x, int) for x in pargs): - # Retrieve a single pixel. - # Something like jp2[r, c] - row = pargs[0] - col = pargs[1] - area = (row, col, row + 1, col + 1) - pixel = self.read(area=area).squeeze() - - if len(pargs) == 2: - return pixel - elif len(pargs) == 3: - return pixel[pargs[2]] - if isinstance(pargs, tuple) and any(x is Ellipsis for x in pargs): nrows = codestream.segment[1].ysiz ncols = codestream.segment[1].xsiz From 9ab1ffff5071e7dd4d86b334e972cfa6a5159202 Mon Sep 17 00:00:00 2001 From: jevans Date: Sun, 5 Oct 2014 21:03:42 -0400 Subject: [PATCH 5/8] add test for two ellipses and one full slice --- glymur/test/test_jp2k.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index 77d796d..9f6f22b 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -271,6 +271,11 @@ class TestSliceProtocolRead(SliceProtocolBase): expected = self.j2k_data[1, :, :] np.testing.assert_array_equal(actual, expected) + def test_two_ellipsis_and_full_slice(self): + actual = self.j2k[..., ..., :] + expected = self.j2k_data[:] + np.testing.assert_array_equal(actual, expected) + def test_slice_protocol_2d_reduce_resolution(self): d = self.j2k[:] self.assertEqual(d.shape, (800, 480, 3)) From 332b3ef00b254e8474e8288cbbd5d2612cef5cbf Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 6 Oct 2014 11:06:44 -0400 Subject: [PATCH 6/8] refactoring --- glymur/jp2k.py | 79 ++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 7b9954e..85dad8a 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -13,10 +13,10 @@ import sys # pylint: disable=E0611 if sys.hexversion >= 0x03030000: from contextlib import ExitStack - from itertools import filterfalse + from itertools import compress, filterfalse else: from contextlib2 import ExitStack - from itertools import ifilterfalse as filterfalse + from itertools import compress, ifilterfalse as filterfalse from collections import Counter import ctypes @@ -779,11 +779,15 @@ class Jp2k(Jp2kBox): Slicing protocol. """ codestream = self.get_codestream(header_only=True) + numrows = codestream.segment[1].ysiz + numcols = codestream.segment[1].xsiz + numbands = codestream.segment[1].Csiz + if isinstance(pargs, int): # Not a very good use of this protocol, but technically legal. # This retrieves a single row. row = pargs - area = (row, 0, row + 1, codestream.segment[1].xsiz) + area = (row, 0, row + 1, numcols) return self.read(area=area).squeeze() #if isinstance(pargs, slice) or pargs is ...: @@ -794,27 +798,33 @@ class Jp2k(Jp2kBox): return self.read() if isinstance(pargs, tuple) and any(x is Ellipsis for x in pargs): - nrows = codestream.segment[1].ysiz - ncols = codestream.segment[1].xsiz - nbands = codestream.segment[1].Csiz - # Reformulate without the ellipsis. + # Search out and remove any Ellipsis objects. + lst = list(pargs) + predicate = lambda x: x[1] is not Ellipsis + g = filterfalse(predicate, enumerate(pargs)) + + rows = slice(0, numrows) + cols = slice(0, numcols) + bands = slice(0, numbands) if pargs[0] is Ellipsis: if len(pargs) == 2: - newindex = (slice(0, nrows), slice(0, ncols), pargs[1]) + newindex = (rows, cols, pargs[1]) else: - newindex = (slice(0, nrows), pargs[1], pargs[2]) + newindex = (rows, pargs[1], pargs[2]) elif pargs[1] is Ellipsis: if len(pargs) == 2: - newindex = (pargs[0], slice(0, ncols), slice(0, nbands)) + newindex = (pargs[0], cols, bands) else: - newindex = (pargs[0], slice(0, ncols), pargs[2]) + newindex = (pargs[0], cols, pargs[2]) else: - newindex = (pargs[0], pargs[1], slice(0, nbands)) + newindex = (pargs[0], pargs[1], bands) + # Run once again because it is possible that there's another + # Ellipsis object in the 2nd or 3rd position. return self.__getitem__(newindex) - if isinstance(pargs, tuple) and not all(isinstance(x, slice) for x in pargs): - # Search out any remaining non-slices and turn them into slices. + if isinstance(pargs, tuple) and any(isinstance(x, int) for x in pargs): + # Replace the first such integer argument, replace it with a slice. lst = list(pargs) predicate = lambda x: not isinstance(x[1], int) g = filterfalse(predicate, enumerate(pargs)) @@ -822,6 +832,8 @@ class Jp2k(Jp2kBox): lst[idx] = slice(pargs[idx], pargs[idx] + 1) newindex = tuple(lst) + # Invoke array-based slicing again, as there may be additional + # integer argument remaining. data = self.__getitem__(newindex) # Reduce dimensionality in the scalar dimension. @@ -836,16 +848,8 @@ class Jp2k(Jp2kBox): else: bands = pargs[2] - if rows.step is None: - rows_step = 1 - else: - rows_step = rows.step - - if cols.step is None: - cols_step = 1 - else: - cols_step = cols.step - + rows_step = 1 if rows.step is None else rows.step + cols_step = 1 if cols.step is None else cols.step if rows_step != cols_step: msg = "Row and column strides must be the same." raise IndexError(msg) @@ -860,27 +864,12 @@ class Jp2k(Jp2kBox): raise IndexError(msg) rlevel = np.int(np.round(np.log2(step))) - if rows.start is None: - rows_start = 0 - else: - rows_start = rows.start - - if rows.stop is None: - rows_stop = codestream.segment[1].ysiz - else: - rows_stop = rows.stop - - if cols.start is None: - cols_start = 0 - else: - cols_start = cols.start - - if cols.stop is None: - cols_stop = codestream.segment[1].xsiz - else: - cols_stop = cols.stop - - area = (rows_start, cols_start, rows_stop, cols_stop) + area = ( + 0 if rows.start is None else rows.start, + 0 if cols.start is None else cols.start, + numrows if rows.stop is None else rows.stop, + numcols if cols.stop is None else cols.stop + ) data = self.read(area=area, rlevel=rlevel) if len(pargs) == 2: return data From e565a71e88303c3ea5afeafb576cd0d00e8f528e Mon Sep 17 00:00:00 2001 From: John Evans Date: Mon, 6 Oct 2014 11:30:56 -0400 Subject: [PATCH 7/8] add test for issue 268 --- glymur/test/test_jp2k.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/glymur/test/test_jp2k.py b/glymur/test/test_jp2k.py index 9f6f22b..042a681 100644 --- a/glymur/test/test_jp2k.py +++ b/glymur/test/test_jp2k.py @@ -276,6 +276,12 @@ class TestSliceProtocolRead(SliceProtocolBase): expected = self.j2k_data[:] np.testing.assert_array_equal(actual, expected) + def test_single_slice(self): + rows = slice(3, 8) + actual = self.j2k[rows] + expected = self.j2k_data[3:8, :,:] + np.testing.assert_array_equal(actual, expected) + def test_slice_protocol_2d_reduce_resolution(self): d = self.j2k[:] self.assertEqual(d.shape, (800, 480, 3)) From ec15eb5ad0569c15139a83962fd2efc595e5d2b5 Mon Sep 17 00:00:00 2001 From: jevans Date: Mon, 6 Oct 2014 19:46:19 -0400 Subject: [PATCH 8/8] fix for case of single slice with start, stop or step that is not none --- glymur/jp2k.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/glymur/jp2k.py b/glymur/jp2k.py index 85dad8a..d460ea6 100644 --- a/glymur/jp2k.py +++ b/glymur/jp2k.py @@ -790,19 +790,22 @@ class Jp2k(Jp2kBox): area = (row, 0, row + 1, numcols) return self.read(area=area).squeeze() - #if isinstance(pargs, slice) or pargs is ...: - if isinstance(pargs, slice) or pargs is Ellipsis: - # Case of jp2[:] or jp2[...], i.e. retrieve the entire image. - # - # Should have a slice object where start = stop = step = None + if pargs is Ellipsis: + # Case of jp2[...] return self.read() - if isinstance(pargs, tuple) and any(x is Ellipsis for x in pargs): - # Search out and remove any Ellipsis objects. - lst = list(pargs) - predicate = lambda x: x[1] is not Ellipsis - g = filterfalse(predicate, enumerate(pargs)) + if isinstance(pargs, slice): + if pargs.start is None and pargs.stop is None and pargs.step is None: + # Case of jp2[:] + return self.read() + # Corner case of jp2[x] where x is a slice object with non-null + # members. Just augment it with an ellipsis and let the code + # below handle it. + pargs = (pargs, Ellipsis) + + if isinstance(pargs, tuple) and any(x is Ellipsis for x in pargs): + # Remove the first ellipsis we find. rows = slice(0, numrows) cols = slice(0, numcols) bands = slice(0, numbands) @@ -817,6 +820,7 @@ class Jp2k(Jp2kBox): else: newindex = (pargs[0], cols, pargs[2]) else: + # Assume that we don't have 4D imagery, of course. newindex = (pargs[0], pargs[1], bands) # Run once again because it is possible that there's another @@ -828,7 +832,7 @@ class Jp2k(Jp2kBox): lst = list(pargs) predicate = lambda x: not isinstance(x[1], int) g = filterfalse(predicate, enumerate(pargs)) - idx = list(g)[0][0] + idx = next(g)[0] lst[idx] = slice(pargs[idx], pargs[idx] + 1) newindex = tuple(lst)