From b3cb55f82360bf39eebb363d8f65cfd97bebf5fe Mon Sep 17 00:00:00 2001 From: Alexey Shamrin Date: Wed, 15 Dec 2021 22:35:32 +0200 Subject: [PATCH 01/54] keep IPython/Jupyter text/plain output stable --- src/PIL/Image.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 0fca3fa5c..2653c6274 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -644,6 +644,19 @@ class Image: id(self), ) + def _repr_pretty_(self, p, cycle): + """IPython plain text display support""" + + # Same as __repr__ but without unpredicatable id(self), + # to keep Jupyter notebook `text/plain` output stable. + p.text("<%s.%s image mode=%s size=%dx%d>" % ( + self.__class__.__module__, + self.__class__.__name__, + self.mode, + self.size[0], + self.size[1], + )) + def _repr_png_(self): """iPython display hook support From 56d630294c31ea7c1360a9830ad82d9303ea6602 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Dec 2021 20:39:38 +0000 Subject: [PATCH 02/54] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/PIL/Image.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 2653c6274..3677cccc3 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -649,13 +649,16 @@ class Image: # Same as __repr__ but without unpredicatable id(self), # to keep Jupyter notebook `text/plain` output stable. - p.text("<%s.%s image mode=%s size=%dx%d>" % ( - self.__class__.__module__, - self.__class__.__name__, - self.mode, - self.size[0], - self.size[1], - )) + p.text( + "<%s.%s image mode=%s size=%dx%d>" + % ( + self.__class__.__module__, + self.__class__.__name__, + self.mode, + self.size[0], + self.size[1], + ) + ) def _repr_png_(self): """iPython display hook support From ba86d90a009621c052e0f268bb0062932b2096f3 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 12 Jan 2022 15:17:04 +1100 Subject: [PATCH 03/54] Simplified code --- src/PIL/SpiderImagePlugin.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/PIL/SpiderImagePlugin.py b/src/PIL/SpiderImagePlugin.py index 062af9f98..061efe8ec 100644 --- a/src/PIL/SpiderImagePlugin.py +++ b/src/PIL/SpiderImagePlugin.py @@ -238,14 +238,14 @@ def makeSpiderHeader(im): if 1024 % lenbyt != 0: labrec += 1 labbyt = labrec * lenbyt - hdr = [] nvalues = int(labbyt / 4) + if nvalues < 23: + return [] + + hdr = [] for i in range(nvalues): hdr.append(0.0) - if len(hdr) < 23: - return [] - # NB these are Fortran indices hdr[1] = 1.0 # nslice (=1 for an image) hdr[2] = float(nrow) # number of rows per slice @@ -259,10 +259,7 @@ def makeSpiderHeader(im): hdr = hdr[1:] hdr.append(0.0) # pack binary data into a string - hdrstr = [] - for v in hdr: - hdrstr.append(struct.pack("f", v)) - return hdrstr + return [struct.pack("f", v) for v in hdr] def _save(im, fp, filename): From a9441a475d4c4133f161b7927352b4bda5cc9b6f Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 12 Jan 2022 17:29:25 +1100 Subject: [PATCH 04/54] Added IREC header entry for use with Bio-formats library --- src/PIL/SpiderImagePlugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PIL/SpiderImagePlugin.py b/src/PIL/SpiderImagePlugin.py index 061efe8ec..11c706b4d 100644 --- a/src/PIL/SpiderImagePlugin.py +++ b/src/PIL/SpiderImagePlugin.py @@ -249,6 +249,7 @@ def makeSpiderHeader(im): # NB these are Fortran indices hdr[1] = 1.0 # nslice (=1 for an image) hdr[2] = float(nrow) # number of rows per slice + hdr[3] = float(nrow) # number of records in the image hdr[5] = 1.0 # iform for 2D image hdr[12] = float(nsam) # number of pixels per line hdr[13] = float(labrec) # number of records in file header From 86944abbabad62e53e644bd7375b9a56d66c1675 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 15 Jan 2022 16:08:37 +1100 Subject: [PATCH 05/54] Deprecated show_file "file" argument in favour of "path" --- Tests/test_imageshow.py | 15 +++++++++++ src/PIL/ImageShow.py | 59 +++++++++++++++++++++++++++++++---------- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index 5981e22c0..02edfdfa1 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -79,3 +79,18 @@ def test_ipythonviewer(): im = hopper() assert test_viewer.show(im) == 1 + + +@pytest.mark.skipif( + not on_ci() or is_win32(), + reason="Only run on CIs; hangs on Windows CIs", +) +def test_file_deprecated(): + for viewer in ImageShow._viewers: + with pytest.warns(DeprecationWarning): + try: + viewer.show_file(file="test.jpg") + except NotImplementedError: + pass + with pytest.raises(TypeError): + viewer.show_file() diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 2135293e5..068b7f784 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -16,6 +16,7 @@ import shutil import subprocess import sys import tempfile +import warnings from shlex import quote from PIL import Image @@ -105,9 +106,19 @@ class Viewer: """Display the given image.""" return self.show_file(self.save_image(image), **options) - def show_file(self, file, **options): - """Display the given file.""" - os.system(self.get_command(file, **options)) + def show_file(self, path=None, **options): + """Display given file.""" + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + os.system(self.get_command(path, **options)) return 1 @@ -145,18 +156,28 @@ class MacViewer(Viewer): command = f"({command} {quote(file)}; sleep 20; rm -f {quote(file)})&" return command - def show_file(self, file, **options): + def show_file(self, path=None, **options): """Display given file""" - fd, path = tempfile.mkstemp() + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + fd, temp_path = tempfile.mkstemp() with os.fdopen(fd, "w") as f: - f.write(file) - with open(path) as f: + f.write(path) + with open(temp_path) as f: subprocess.Popen( ["im=$(cat); open -a Preview.app $im; sleep 20; rm -f $im"], shell=True, stdin=f, ) - os.remove(path) + os.remove(temp_path) return 1 @@ -172,17 +193,27 @@ class UnixViewer(Viewer): command = self.get_command_ex(file, **options)[0] return f"({command} {quote(file)}; rm -f {quote(file)})&" - def show_file(self, file, **options): + def show_file(self, path=None, **options): """Display given file""" - fd, path = tempfile.mkstemp() + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + fd, temp_path = tempfile.mkstemp() with os.fdopen(fd, "w") as f: - f.write(file) - with open(path) as f: - command = self.get_command_ex(file, **options)[0] + f.write(path) + with open(temp_path) as f: + command = self.get_command_ex(path, **options)[0] subprocess.Popen( ["im=$(cat);" + command + " $im; rm -f $im"], shell=True, stdin=f ) - os.remove(path) + os.remove(temp_path) return 1 From 26496a6c981bac4ab035369ee591616f31dcc67a Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 16 Jan 2022 10:52:22 +1100 Subject: [PATCH 06/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 49ab72962..3e8c72dd8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Fixed Spider images for use with Bio-formats library #5956 + [radarhere] + - Ensure duplicated file pointer is closed #5946 [radarhere] From 5df83a57ff0f8362fc1d66180f777d5d79672342 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 16 Jan 2022 11:38:34 +1100 Subject: [PATCH 07/54] Documented deprecation --- docs/deprecations.rst | 12 ++++++++++++ src/PIL/ImageShow.py | 21 ++++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/deprecations.rst b/docs/deprecations.rst index ce30fdf3b..dbe65dda1 100644 --- a/docs/deprecations.rst +++ b/docs/deprecations.rst @@ -53,6 +53,18 @@ Before Pillow 8.3.0, ``ImagePalette`` required palette data of particular length default, and the size parameter could be used to override that. Pillow 8.3.0 removed the default required length, also removing the need for the size parameter. +ImageShow.Viewer.show_file file argument +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 9.1.0 + +The ``file`` argument in :py:meth:`~PIL.ImageShow.Viewer.show_file()` has been +deprecated, replaced by ``path``. + +In effect, ``viewer.show_file("test.jpg")`` will continue to work unchanged. +``viewer.show_file(file="test.jpg")`` will raise a deprecation warning, and suggest +``viewer.show_file(path="test.jpg")`` instead. + Removed features ---------------- diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 068b7f784..e437bbade 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -107,7 +107,12 @@ class Viewer: return self.show_file(self.save_image(image), **options) def show_file(self, path=None, **options): - """Display given file.""" + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ if path is None: if "file" in options: warnings.warn( @@ -157,7 +162,12 @@ class MacViewer(Viewer): return command def show_file(self, path=None, **options): - """Display given file""" + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ if path is None: if "file" in options: warnings.warn( @@ -194,7 +204,12 @@ class UnixViewer(Viewer): return f"({command} {quote(file)}; rm -f {quote(file)})&" def show_file(self, path=None, **options): - """Display given file""" + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ if path is None: if "file" in options: warnings.warn( From 1ac96dc923a5f2360348b4e00b070856fff8a78e Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 16 Jan 2022 14:09:12 +1100 Subject: [PATCH 08/54] Invoke pip using Python --- .ci/after_success.sh | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/after_success.sh b/.ci/after_success.sh index ff91b481e..53832c573 100755 --- a/.ci/after_success.sh +++ b/.ci/after_success.sh @@ -1,7 +1,7 @@ #!/bin/bash # gather the coverage data -pip3 install codecov +python3 -m pip install codecov if [[ $MATRIX_DOCKER ]]; then coverage xml --ignore-errors else diff --git a/Makefile b/Makefile index 44ab2ef1c..74a6a5ab2 100644 --- a/Makefile +++ b/Makefile @@ -105,7 +105,7 @@ test: .PHONY: valgrind valgrind: - python3 -c "import pytest_valgrind" || pip3 install pytest-valgrind + python3 -c "import pytest_valgrind" || python3 -m pip install pytest-valgrind PYTHONMALLOC=malloc valgrind --suppressions=Tests/oss-fuzz/python.supp --leak-check=no \ --log-file=/tmp/valgrind-output \ python3 -m pytest --no-memcheck -vv --valgrind --valgrind-log=/tmp/valgrind-output From 730e24e163ea6b891f24efe4dcb1869d85c2f63c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 17 Jan 2022 07:35:11 +1100 Subject: [PATCH 09/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3e8c72dd8..b3379c1ee 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,10 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ -- Fixed Spider images for use with Bio-formats library #5956 +- Deprecated show_file "file" argument in favour of "path" #5959 + [radarhere] + +- Fixed SPIDER images for use with Bio-formats library #5956 [radarhere] - Ensure duplicated file pointer is closed #5946 From b53cb3f675a2987986ff4f794df123a44fa6181c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 17 Jan 2022 09:47:27 +1100 Subject: [PATCH 10/54] Updated macOS tested Pillow versions [ci skip] --- docs/installation.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index df21a7cdc..339ff966d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -493,9 +493,13 @@ These platforms have been reported to work at the versions mentioned. | Operating system | | Tested Python | | Latest tested | | Tested | | | | versions | | Pillow version | | processors | +==================================+===========================+==================+==============+ -| macOS 11.0 Big Sur | 3.7, 3.8, 3.9, 3.10 | 8.4.0 |arm | +| macOS 12 Big Sur | 3.7, 3.8, 3.9, 3.10 | 9.0.0 |arm | ++----------------------------------+---------------------------+------------------+--------------+ +| macOS 11 Big Sur | 3.7, 3.8, 3.9, 3.10 | 8.4.0 |arm | | +---------------------------+------------------+--------------+ -| | 3.6, 3.7, 3.8, 3.9, 3.10 | 8.4.0 |x86-64 | +| | 3.7, 3.8, 3.9, 3.10 | 9.0.0 |x86-64 | +| +---------------------------+------------------+--------------+ +| | 3.6 | 8.4.0 |x86-64 | +----------------------------------+---------------------------+------------------+--------------+ | macOS 10.15 Catalina | 3.6, 3.7, 3.8, 3.9 | 8.3.2 |x86-64 | | +---------------------------+------------------+ | From c8d650f38303d17af0c4aefbb9ee9681e26fabea Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 17 Jan 2022 14:07:30 +1100 Subject: [PATCH 11/54] Added Debian 11 Bullseye --- .github/workflows/test-docker.yml | 1 + docs/installation.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 57396fddc..df04d0a6c 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -22,6 +22,7 @@ jobs: centos-8-amd64, centos-stream-8-amd64, debian-10-buster-x86, + debian-11-bullseye-x86, fedora-34-amd64, fedora-35-amd64, ubuntu-18.04-bionic-amd64, diff --git a/docs/installation.rst b/docs/installation.rst index df21a7cdc..3b249b368 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -458,6 +458,8 @@ These platforms are built and tested for every change. +----------------------------------+----------------------------+---------------------+ | Debian 10 Buster | 3.7 | x86 | +----------------------------------+----------------------------+---------------------+ +| Debian 11 Bullseye | 3.9 | x86 | ++----------------------------------+----------------------------+---------------------+ | Fedora 34 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ | Fedora 35 | 3.10 | x86-64 | From ddb0a6393f3fcb4aeb78f0567a58d9058ba9ca72 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 18 Jan 2022 11:46:29 +1100 Subject: [PATCH 12/54] Added test --- Tests/test_image.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Tests/test_image.py b/Tests/test_image.py index cf60f42af..db3e3063e 100644 --- a/Tests/test_image.py +++ b/Tests/test_image.py @@ -88,6 +88,17 @@ class TestImage: # with pytest.raises(MemoryError): # Image.new("L", (1000000, 1000000)) + def test_repr_pretty(self): + class Pretty: + def text(self, text): + self.pretty_output = text + + im = Image.new("L", (100, 100)) + + p = Pretty() + im._repr_pretty_(p, None) + assert p.pretty_output == "" + def test_open_formats(self): PNGFILE = "Tests/images/hopper.png" JPGFILE = "Tests/images/hopper.jpg" From a77babd6a459b9e1650bddc59ce6e3edf226f198 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 18 Jan 2022 14:26:52 +1100 Subject: [PATCH 13/54] Updated links --- README.md | 4 ++-- docs/about.rst | 2 +- docs/index.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 782b81f33..63671f405 100644 --- a/README.md +++ b/README.md @@ -42,10 +42,10 @@ As of 2019, Pillow development is GitHub Actions wheels build status (Wheels) - Travis CI wheels build status (aarch64) - Code coverage Date: Tue, 18 Jan 2022 16:38:00 +1100 Subject: [PATCH 14/54] Raise an error when performing a negative crop --- Tests/test_decompression_bomb.py | 17 ++++------------- Tests/test_image_crop.py | 14 +++++--------- src/PIL/Image.py | 5 +++++ 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/Tests/test_decompression_bomb.py b/Tests/test_decompression_bomb.py index d918ef941..1778491ab 100644 --- a/Tests/test_decompression_bomb.py +++ b/Tests/test_decompression_bomb.py @@ -86,21 +86,12 @@ class TestDecompressionCrop: pytest.warns(Image.DecompressionBombWarning, src.crop, box) def test_crop_decompression_checks(self): - im = Image.new("RGB", (100, 100)) - good_values = ((-9999, -9999, -9990, -9990), (-999, -999, -990, -990)) - - warning_values = ((-160, -160, 99, 99), (160, 160, -99, -99)) - - error_values = ((-99909, -99990, 99999, 99999), (99909, 99990, -99999, -99999)) - - for value in good_values: + for value in ((-9999, -9999, -9990, -9990), (-999, -999, -990, -990)): assert im.crop(value).size == (9, 9) - for value in warning_values: - pytest.warns(Image.DecompressionBombWarning, im.crop, value) + pytest.warns(Image.DecompressionBombWarning, im.crop, (-160, -160, 99, 99)) - for value in error_values: - with pytest.raises(Image.DecompressionBombError): - im.crop(value) + with pytest.raises(Image.DecompressionBombError): + im.crop((-99909, -99990, 99999, 99999)) diff --git a/Tests/test_image_crop.py b/Tests/test_image_crop.py index e2228758c..6574e6efd 100644 --- a/Tests/test_image_crop.py +++ b/Tests/test_image_crop.py @@ -47,16 +47,12 @@ def test_wide_crop(): assert crop(-25, 75, 25, 125) == (1875, 625) -def test_negative_crop(): - # Check negative crop size (@PIL171) +@pytest.mark.parametrize("box", ((8, 2, 2, 8), (2, 8, 8, 2), (8, 8, 2, 2))) +def test_negative_crop(box): + im = Image.new("RGB", (10, 10)) - im = Image.new("L", (512, 512)) - im = im.crop((400, 400, 200, 200)) - - assert im.size == (0, 0) - assert len(im.getdata()) == 0 - with pytest.raises(IndexError): - im.getdata()[0] + with pytest.raises(ValueError): + im.crop(box) def test_crop_float(): diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 69089a290..439ffc65e 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -1145,6 +1145,11 @@ class Image: if box is None: return self.copy() + if box[2] < box[0]: + raise ValueError("Region right less than region left") + elif box[3] < box[1]: + raise ValueError("Region lower less than region upper") + self.load() return self._new(self._crop(self.im, box)) From ef99a73473aed8860d85b2979f2928e0f1e8c4ed Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 20 Jan 2022 10:04:35 +1100 Subject: [PATCH 15/54] Clarified that version numbers refer to Tk, not Pillow --- src/Tk/tkImaging.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Tk/tkImaging.c b/src/Tk/tkImaging.c index 9ae7edff1..5b3f18ace 100644 --- a/src/Tk/tkImaging.c +++ b/src/Tk/tkImaging.c @@ -29,7 +29,7 @@ * 1995-09-12 fl Created * 1996-04-08 fl Ready for release * 1997-05-09 fl Use command instead of image type - * 2001-03-18 fl Initialize alpha layer pointer (struct changed in 8.3) + * 2001-03-18 fl Initialize alpha layer pointer (struct changed in Tk 8.3) * 2003-04-23 fl Fixed building for Tk 8.4.1 and later (Jack Jansen) * 2004-06-24 fl Fixed building for Tk 8.4.6 and later. * @@ -116,7 +116,7 @@ PyImagingPhotoPut( block.offset[1] = 1; block.offset[2] = 2; if (strcmp(im->mode, "RGBA") == 0) { - block.offset[3] = 3; /* alpha (or reserved, under 8.2) */ + block.offset[3] = 3; /* alpha (or reserved, under Tk 8.2) */ } else { block.offset[3] = 0; /* no alpha */ } From 591231bbb408985d2b8ca58809fd237d704ffd1d Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Fri, 21 Jan 2022 08:19:27 +1100 Subject: [PATCH 16/54] Changed error wording Co-authored-by: Hugo van Kemenade --- src/PIL/Image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/PIL/Image.py b/src/PIL/Image.py index 439ffc65e..302bd638e 100644 --- a/src/PIL/Image.py +++ b/src/PIL/Image.py @@ -1146,9 +1146,9 @@ class Image: return self.copy() if box[2] < box[0]: - raise ValueError("Region right less than region left") + raise ValueError("Coordinate 'right' is less than 'left'") elif box[3] < box[1]: - raise ValueError("Region lower less than region upper") + raise ValueError("Coordinate 'lower' is less than 'upper'") self.load() return self._new(self._crop(self.im, box)) From 4eb80cb6bfb2f83dc039fb5bbce56a4416bac30f Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 22 Jan 2022 09:14:01 +1100 Subject: [PATCH 17/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b3379c1ee..d6cd9d50f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Raise an error when performing a negative crop #5972 + [radarhere, hugovk] + - Deprecated show_file "file" argument in favour of "path" #5959 [radarhere] From 54f85ddcade1c93edb84b47796cc178f37c7e8fd Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sat, 22 Jan 2022 09:30:12 +1100 Subject: [PATCH 18/54] Updated libwebp to 1.2.2 --- depends/install_webp.sh | 2 +- winbuild/build_prepare.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/depends/install_webp.sh b/depends/install_webp.sh index 8a9c96804..a419a7646 100755 --- a/depends/install_webp.sh +++ b/depends/install_webp.sh @@ -1,7 +1,7 @@ #!/bin/bash # install webp -archive=libwebp-1.2.1 +archive=libwebp-1.2.2 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 0589baf21..3092c5a2b 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -154,9 +154,9 @@ deps = { # "bins": [r"libtiff\*.dll"], }, "libwebp": { - "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.2.1.tar.gz", - "filename": "libwebp-1.2.1.tar.gz", - "dir": "libwebp-1.2.1", + "url": "http://downloads.webmproject.org/releases/webp/libwebp-1.2.2.tar.gz", + "filename": "libwebp-1.2.2.tar.gz", + "dir": "libwebp-1.2.2", "build": [ cmd_rmdir(r"output\release-static"), # clean cmd_nmake( From 31aa2ad98cffb8fd8abd05a34e56ff7f1094b553 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 24 Jan 2022 11:06:41 +1100 Subject: [PATCH 19/54] Removed unused variables --- Tests/test_file_psd.py | 2 +- src/PIL/PsdImagePlugin.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Tests/test_file_psd.py b/Tests/test_file_psd.py index f50fe133f..a7f379e55 100644 --- a/Tests/test_file_psd.py +++ b/Tests/test_file_psd.py @@ -123,7 +123,7 @@ def test_no_icc_profile(): def test_combined_larger_than_size(): - # The 'combined' sizes of the individual parts is larger than the + # The combined size of the individual parts is larger than the # declared 'size' of the extra data field, resulting in a backwards seek. # If we instead take the 'size' of the extra data field as the source of truth, diff --git a/src/PIL/PsdImagePlugin.py b/src/PIL/PsdImagePlugin.py index 04b21e3de..550a333dd 100644 --- a/src/PIL/PsdImagePlugin.py +++ b/src/PIL/PsdImagePlugin.py @@ -195,7 +195,6 @@ def _layerinfo(fp, ct_bytes): x1 = i32(read(4)) # image info - info = [] mode = [] ct_types = i16(read(2)) types = list(range(ct_types)) @@ -211,8 +210,7 @@ def _layerinfo(fp, ct_bytes): m = "RGBA"[type] mode.append(m) - size = i32(read(4)) - info.append((m, size)) + read(4) # size # figure out the image mode mode.sort() @@ -229,26 +227,22 @@ def _layerinfo(fp, ct_bytes): read(12) # filler name = "" size = i32(read(4)) # length of the extra data field - combined = 0 if size: data_end = fp.tell() + size length = i32(read(4)) if length: fp.seek(length - 16, io.SEEK_CUR) - combined += length + 4 length = i32(read(4)) if length: fp.seek(length, io.SEEK_CUR) - combined += length + 4 length = i8(read(1)) if length: # Don't know the proper encoding, # Latin-1 should be a good guess name = read(length).decode("latin-1", "replace") - combined += length + 1 fp.seek(data_end) layers.append((name, mode, (x0, y0, x1, y1))) From 1d91f6dce5732c48710ad73f8489568471be449d Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 26 Jan 2022 09:08:41 +1100 Subject: [PATCH 20/54] Document when file argument will be removed --- docs/deprecations.rst | 3 ++- src/PIL/ImageShow.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/deprecations.rst b/docs/deprecations.rst index dbe65dda1..a3abe81fa 100644 --- a/docs/deprecations.rst +++ b/docs/deprecations.rst @@ -59,7 +59,8 @@ ImageShow.Viewer.show_file file argument .. deprecated:: 9.1.0 The ``file`` argument in :py:meth:`~PIL.ImageShow.Viewer.show_file()` has been -deprecated, replaced by ``path``. +deprecated and will be removed in Pillow 10.0.0 (2023-07-01). It has been replaced by +``path``. In effect, ``viewer.show_file("test.jpg")`` will continue to work unchanged. ``viewer.show_file(file="test.jpg")`` will raise a deprecation warning, and suggest diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index e437bbade..2165da307 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -111,7 +111,8 @@ class Viewer: Display given file. Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, - and ``path`` should be used instead. + and will be removed in Pillow 10.0.0 (2023-07-01). ``path`` should be used + instead. """ if path is None: if "file" in options: @@ -166,7 +167,8 @@ class MacViewer(Viewer): Display given file. Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, - and ``path`` should be used instead. + and will be removed in Pillow 10.0.0 (2023-07-01). ``path`` should be used + instead. """ if path is None: if "file" in options: @@ -208,7 +210,8 @@ class UnixViewer(Viewer): Display given file. Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, - and ``path`` should be used instead. + and will be removed in Pillow 10.0.0 (2023-07-01). ``path`` should be used + instead. """ if path is None: if "file" in options: From 93008c20957a7ae94fbd7b03da1f2980d4abb50c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 26 Jan 2022 12:23:43 +1100 Subject: [PATCH 21/54] Upgraded raqm to 0.8.0 --- .ci/install.sh | 2 +- depends/install_raqm.sh | 4 ++-- docs/installation.rst | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/install.sh b/.ci/install.sh index c48acf9ee..efc57a641 100755 --- a/.ci/install.sh +++ b/.ci/install.sh @@ -19,7 +19,7 @@ set -e sudo apt-get -qq install libfreetype6-dev liblcms2-dev python3-tk\ ghostscript libffi-dev libjpeg-turbo-progs libopenjp2-7-dev\ - cmake imagemagick libharfbuzz-dev libfribidi-dev + cmake meson imagemagick libharfbuzz-dev libfribidi-dev python3 -m pip install --upgrade pip python3 -m pip install --upgrade wheel diff --git a/depends/install_raqm.sh b/depends/install_raqm.sh index 3105465ec..39b998d05 100755 --- a/depends/install_raqm.sh +++ b/depends/install_raqm.sh @@ -2,13 +2,13 @@ # install raqm -archive=raqm-0.7.1 +archive=libraqm-0.8.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz pushd $archive -./configure --prefix=/usr && make -j4 && sudo make -j4 install +meson build --prefix=/usr && sudo ninja -C build install popd diff --git a/docs/installation.rst b/docs/installation.rst index 0699ef341..ea0e78500 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -394,7 +394,8 @@ Prerequisites for **Ubuntu 16.04 LTS - 20.04 LTS** are installed with:: libfreetype6-dev liblcms2-dev libwebp-dev tcl8.6-dev tk8.6-dev python3-tk \ libharfbuzz-dev libfribidi-dev libxcb1-dev -Then see ``depends/install_raqm.sh`` to install libraqm. +To install libraqm, ``sudo apt-get install meson`` and then see +``depends/install_raqm.sh``. Prerequisites are installed on recent **Red Hat**, **CentOS** or **Fedora** with:: From 5848726cadd40ff5be20497d7e8168cd4acf1d07 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 28 Jan 2022 11:23:51 +1100 Subject: [PATCH 22/54] Link directly to workflow pages --- README.md | 8 ++++---- docs/index.rst | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 63671f405..a0cb25d32 100644 --- a/README.md +++ b/README.md @@ -24,16 +24,16 @@ As of 2019, Pillow development is tests - GitHub Actions build status (Lint) - GitHub Actions build status (Test Linux and macOS) - GitHub Actions build status (Test Windows) - GitHub Actions build status (Test Docker) Date: Fri, 28 Jan 2022 11:25:24 +1100 Subject: [PATCH 23/54] Added MinGW GitHub Actions badge --- README.md | 3 +++ docs/index.rst | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index a0cb25d32..7bff737a2 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,9 @@ As of 2019, Pillow development is GitHub Actions build status (Test Windows) + GitHub Actions build status (Test MinGW) GitHub Actions build status (Test Docker) diff --git a/docs/index.rst b/docs/index.rst index c57a779df..f1a721c6a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,10 @@ Pillow for enterprise is available via the Tidelift Subscription. `Learn more Date: Fri, 28 Jan 2022 22:04:57 +1100 Subject: [PATCH 24/54] Updated raqm to 0.8.0 --- src/thirdparty/raqm/COPYING | 2 +- src/thirdparty/raqm/NEWS | 16 +- src/thirdparty/raqm/README.md | 27 +- src/thirdparty/raqm/raqm-version.h | 6 +- src/thirdparty/raqm/raqm.c | 401 ++++++++++++++++++++++------- src/thirdparty/raqm/raqm.h | 9 +- 6 files changed, 348 insertions(+), 113 deletions(-) diff --git a/src/thirdparty/raqm/COPYING b/src/thirdparty/raqm/COPYING index 196511ef6..78db981bb 100644 --- a/src/thirdparty/raqm/COPYING +++ b/src/thirdparty/raqm/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright © 2015 Information Technology Authority (ITA) -Copyright © 2016 Khaled Hosny +Copyright © 2016-2021 Khaled Hosny Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/thirdparty/raqm/NEWS b/src/thirdparty/raqm/NEWS index c49176a95..ae1128485 100644 --- a/src/thirdparty/raqm/NEWS +++ b/src/thirdparty/raqm/NEWS @@ -1,4 +1,18 @@ -Overview of changes leading to 0.7.1 +Overview of changes leading to 0.8.0 +Monday, December 13, 2021 +==================================== + +Remove autotools build. + +Support using SheenBiDi instead of FriBiDi for Unicode BiDi support. + +Fix running tests with Python <= 3.6. + +New API: + * raqm_get_par_resolved_direction + * raqm_get_direction_at_index + +Overview of changes leading to 0.7.2 Monday, September 27, 2021 ==================================== diff --git a/src/thirdparty/raqm/README.md b/src/thirdparty/raqm/README.md index 64937343a..17ce2efc9 100644 --- a/src/thirdparty/raqm/README.md +++ b/src/thirdparty/raqm/README.md @@ -6,26 +6,26 @@ Raqm Raqm is a small library that encapsulates the logic for complex text layout and provides a convenient API. -It currently provides bidirectional text support (using [FriBiDi][1]), shaping -(using [HarfBuzz][2]), and proper script itemization. As a result, -Raqm can support most writing systems covered by Unicode. +It currently provides bidirectional text support (using [FriBiDi][1] or +[SheenBidi][2]), shaping (using [HarfBuzz][3]), and proper script itemization. +As a result, Raqm can support most writing systems covered by Unicode. The documentation can be accessed on the web at: > http://host-oman.github.io/libraqm/ Raqm (Arabic: رَقْم) is writing, also number or digit and the Arabic word for -digital (رَقَمِيّ) shares the same root, so it is a play on “digital writing”. +digital (رَقَمِيّ) shares the same root, so it is a play on “digital writing”. Building -------- Raqm depends on the following libraries: -* [FreeType][3] -* [HarfBuzz][2] -* [FriBiDi][1] +* [FreeType][4] +* [HarfBuzz][3] +* [FriBiDi][1] or [SheenBidi][2] To build the documentation you will also need: -* [GTK-Doc][4] +* [GTK-Doc][5] To install dependencies on Fedora: @@ -48,11 +48,11 @@ directory: $ ninja -C build $ ninja -C build install -To build the documentation, pass `-Ddocs=enable` to the `meson`. +To build the documentation, pass `-Ddocs=true` to the `meson`. To run the tests: - $ ninja -C test + $ ninja -C build test Contributing ------------ @@ -78,6 +78,7 @@ The following projects have patches to support complex text layout using Raqm: [1]: http://fribidi.org -[2]: http://harfbuzz.org -[3]: https://www.freetype.org -[4]: https://www.gtk.org/gtk-doc +[2]: https://github.com/Tehreer/SheenBidi +[3]: http://harfbuzz.org +[4]: https://www.freetype.org +[5]: https://www.gtk.org/gtk-doc diff --git a/src/thirdparty/raqm/raqm-version.h b/src/thirdparty/raqm/raqm-version.h index 8b115f612..e96b1aea5 100644 --- a/src/thirdparty/raqm/raqm-version.h +++ b/src/thirdparty/raqm/raqm-version.h @@ -32,10 +32,10 @@ #define _RAQM_VERSION_H_ #define RAQM_VERSION_MAJOR 0 -#define RAQM_VERSION_MINOR 7 -#define RAQM_VERSION_MICRO 2 +#define RAQM_VERSION_MINOR 8 +#define RAQM_VERSION_MICRO 0 -#define RAQM_VERSION_STRING "0.7.2" +#define RAQM_VERSION_STRING "0.8.0" #define RAQM_VERSION_ATLEAST(major,minor,micro) \ ((major)*10000+(minor)*100+(micro) <= \ diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index 31161c9d9..46890b9a7 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016 Khaled Hosny + * Copyright © 2016-2021 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -30,11 +30,18 @@ #include #include +#ifdef RAQM_SHEENBIDI +#include +#else #ifdef HAVE_FRIBIDI_SYSTEM #include #else #include "../fribidi-shim/fribidi.h" #endif +#if FRIBIDI_MAJOR_VERSION >= 1 +#define USE_FRIBIDI_EX_API +#endif +#endif #include #include @@ -56,10 +63,6 @@ #include "raqm.h" -#if FRIBIDI_MAJOR_VERSION >= 1 -#define USE_FRIBIDI_EX_API -#endif - /** * SECTION:raqm * @title: Raqm @@ -178,6 +181,15 @@ # define RAQM_TEST(...) #endif +#define RAQM_BIDI_LEVEL_IS_RTL(level) \ + ((level) & 1) + +#ifdef RAQM_SHEENBIDI + typedef SBLevel _raqm_bidi_level_t; +#else + typedef FriBidiLevel _raqm_bidi_level_t; +#endif + typedef enum { RAQM_FLAG_NONE = 0, RAQM_FLAG_UTF8 = 1 << 0 @@ -438,6 +450,53 @@ raqm_set_text (raqm_t *rq, return true; } +static void * +_raqm_get_utf8_codepoint (const void *str, + uint32_t *out_codepoint) +{ + const char *s = (const char *)str; + + if (0xf0 == (0xf8 & s[0])) + { + *out_codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | ((0x3f & s[2]) << 6) | (0x3f & s[3]); + s += 4; + } + else if (0xe0 == (0xf0 & s[0])) + { + *out_codepoint = ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]); + s += 3; + } + else if (0xc0 == (0xe0 & s[0])) + { + *out_codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]); + s += 2; + } + else + { + *out_codepoint = s[0]; + s += 1; + } + + return (void *)s; +} + +static size_t +_raqm_u8_to_u32 (const char *text, size_t len, uint32_t *unicode) +{ + size_t in_len = 0; + uint32_t *out_utf32 = unicode; + const char *in_utf8 = text; + + while ((*in_utf8 != '\0') && (in_len < len)) + { + in_utf8 = _raqm_get_utf8_codepoint (in_utf8, out_utf32); + ++out_utf32; + ++in_len; + } + + return (out_utf32 - unicode); +} + /** * raqm_set_text_utf8: * @rq: a #raqm_t. @@ -482,9 +541,7 @@ raqm_set_text_utf8 (raqm_t *rq, memcpy (rq->text_utf8, text, sizeof (char) * len); - ulen = fribidi_charset_to_unicode (FRIBIDI_CHAR_SET_UTF8, - text, len, unicode); - + ulen = _raqm_u8_to_u32 (text, len, unicode); ok = raqm_set_text (rq, unicode, ulen); free (unicode); @@ -504,7 +561,7 @@ raqm_set_text_utf8 (raqm_t *rq, * * The default is #RAQM_DIRECTION_DEFAULT, which determines the paragraph * direction based on the first character with strong bidi type (see [rule - * P2](https://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), + * P2](http://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), * which can be good enough for many cases but has problems when a mainly * right-to-left paragraph starts with a left-to-right character and vice versa * as the detected paragraph direction will be the wrong one, or when text does @@ -971,17 +1028,78 @@ raqm_get_glyphs (raqm_t *rq, return rq->glyphs; } +/** + * raqm_get_par_resolved_direction: + * @rq: a #raqm_t. + * + * Gets the resolved direction of the paragraph; + * + * Return value: + * The #raqm_direction_t specifying the resolved direction of text, + * or #RAQM_DIRECTION_DEFAULT if raqm_layout() has not been called on @rq. + * + * Since: 0.8 + */ +RAQM_API raqm_direction_t +raqm_get_par_resolved_direction (raqm_t *rq) +{ + if (!rq) + return RAQM_DIRECTION_DEFAULT; + + return rq->resolved_dir; +} + +/** + * raqm_get_direction_at_index: + * @rq: a #raqm_t. + * @index: (in): character index. + * + * Gets the resolved direction of the character at specified index; + * + * Return value: + * The #raqm_direction_t specifying the resolved direction of text at the + * specified index, or #RAQM_DIRECTION_DEFAULT if raqm_layout() has not been + * called on @rq. + * + * Since: 0.8 + */ +RAQM_API raqm_direction_t +raqm_get_direction_at_index (raqm_t *rq, + size_t index) +{ + if (!rq) + return RAQM_DIRECTION_DEFAULT; + + for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) + { + if (run->pos <= index && index < run->pos + run->len) { + switch (run->direction) { + case HB_DIRECTION_LTR: + return RAQM_DIRECTION_LTR; + case HB_DIRECTION_RTL: + return RAQM_DIRECTION_RTL; + case HB_DIRECTION_TTB: + return RAQM_DIRECTION_TTB; + default: + return RAQM_DIRECTION_DEFAULT; + } + } + } + + return RAQM_DIRECTION_DEFAULT; +} + static bool _raqm_resolve_scripts (raqm_t *rq); static hb_direction_t -_raqm_hb_dir (raqm_t *rq, FriBidiLevel level) +_raqm_hb_dir (raqm_t *rq, _raqm_bidi_level_t level) { hb_direction_t dir = HB_DIRECTION_LTR; if (rq->base_dir == RAQM_DIRECTION_TTB) dir = HB_DIRECTION_TTB; - else if (FRIBIDI_LEVEL_IS_RTL (level)) + else if (RAQM_BIDI_LEVEL_IS_RTL(level)) dir = HB_DIRECTION_RTL; return dir; @@ -990,9 +1108,65 @@ _raqm_hb_dir (raqm_t *rq, FriBidiLevel level) typedef struct { size_t pos; size_t len; - FriBidiLevel level; + _raqm_bidi_level_t level; } _raqm_bidi_run; +#ifdef RAQM_SHEENBIDI +static _raqm_bidi_run * +_raqm_bidi_itemize (raqm_t *rq, size_t *run_count) +{ + _raqm_bidi_run *runs; + SBAlgorithmRef bidi; + SBParagraphRef par; + SBUInteger par_len; + SBLineRef line; + + SBLevel base_level = SBLevelDefaultLTR; + SBCodepointSequence input = { + SBStringEncodingUTF32, + (void *) rq->text, + rq->text_len + }; + + if (rq->base_dir == RAQM_DIRECTION_RTL) + base_level = 1; + else if (rq->base_dir == RAQM_DIRECTION_LTR) + base_level = 0; + + /* paragraph */ + bidi = SBAlgorithmCreate (&input); + par = SBAlgorithmCreateParagraph (bidi, 0, INT32_MAX, base_level); + par_len = SBParagraphGetLength (par); + + /* lines */ + line = SBParagraphCreateLine (par, 0, par_len); + *run_count = SBLineGetRunCount (line); + + if (SBParagraphGetBaseLevel (par) == 0) + rq->resolved_dir = RAQM_DIRECTION_LTR; + else + rq->resolved_dir = RAQM_DIRECTION_RTL; + + runs = malloc (sizeof (_raqm_bidi_run) * (*run_count)); + if (runs) + { + const SBRun *sheenbidi_runs = SBLineGetRunsPtr(line); + + for (size_t i = 0; i < (*run_count); ++i) + { + runs[i].pos = sheenbidi_runs[i].offset; + runs[i].len = sheenbidi_runs[i].length; + runs[i].level = sheenbidi_runs[i].level; + } + } + + SBLineRelease (line); + SBParagraphRelease (par); + SBAlgorithmRelease (bidi); + + return runs; +} +#else static void _raqm_reverse_run (_raqm_bidi_run *run, const size_t len) { @@ -1093,19 +1267,78 @@ _raqm_reorder_runs (const FriBidiCharType *types, return runs; } -static bool -_raqm_itemize (raqm_t *rq) +static _raqm_bidi_run * +_raqm_bidi_itemize (raqm_t *rq, size_t *run_count) { FriBidiParType par_type = FRIBIDI_PAR_ON; + _raqm_bidi_run *runs = NULL; + FriBidiCharType *types; + _raqm_bidi_level_t *levels; + int max_level = 0; #ifdef USE_FRIBIDI_EX_API FriBidiBracketType *btypes; #endif - FriBidiLevel *levels; + + types = calloc (rq->text_len, sizeof (FriBidiCharType)); +#ifdef USE_FRIBIDI_EX_API + btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); +#endif + levels = calloc (rq->text_len, sizeof (_raqm_bidi_level_t)); + + if (!types || !levels +#ifdef USE_FRIBIDI_EX_API + || !btypes +#endif + ) + { + goto done; + } + + if (rq->base_dir == RAQM_DIRECTION_RTL) + par_type = FRIBIDI_PAR_RTL; + else if (rq->base_dir == RAQM_DIRECTION_LTR) + par_type = FRIBIDI_PAR_LTR; + + fribidi_get_bidi_types (rq->text, rq->text_len, types); +#ifdef USE_FRIBIDI_EX_API + fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); + max_level = fribidi_get_par_embedding_levels_ex (types, btypes, + rq->text_len, &par_type, + levels); +#else + max_level = fribidi_get_par_embedding_levels (types, rq->text_len, + &par_type, levels); +#endif + + if (par_type == FRIBIDI_PAR_LTR) + rq->resolved_dir = RAQM_DIRECTION_LTR; + else + rq->resolved_dir = RAQM_DIRECTION_RTL; + + if (max_level == 0) + goto done; + + /* Get the number of bidi runs */ + runs = _raqm_reorder_runs (types, rq->text_len, par_type, levels, run_count); + +done: + free (types); + free (levels); +#ifdef USE_FRIBIDI_EX_API + free (btypes); +#endif + + return runs; +} +#endif + +static bool +_raqm_itemize (raqm_t *rq) +{ _raqm_bidi_run *runs = NULL; raqm_run_t *last; - int max_level; - size_t run_count; + size_t run_count = 0; bool ok = true; #ifdef RAQM_TESTING @@ -1127,67 +1360,28 @@ _raqm_itemize (raqm_t *rq) } #endif - types = calloc (rq->text_len, sizeof (FriBidiCharType)); -#ifdef USE_FRIBIDI_EX_API - btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); -#endif - levels = calloc (rq->text_len, sizeof (FriBidiLevel)); - if (!types || !levels -#ifdef USE_FRIBIDI_EX_API - || !btypes -#endif - ) - { - ok = false; - goto done; - } - - if (rq->base_dir == RAQM_DIRECTION_RTL) - par_type = FRIBIDI_PAR_RTL; - else if (rq->base_dir == RAQM_DIRECTION_LTR) - par_type = FRIBIDI_PAR_LTR; - - if (rq->base_dir == RAQM_DIRECTION_TTB) - { - /* Treat every thing as LTR in vertical text */ - max_level = 1; - memset (types, FRIBIDI_TYPE_LTR, rq->text_len); - memset (levels, 0, rq->text_len); - rq->resolved_dir = RAQM_DIRECTION_LTR; - } - else - { - fribidi_get_bidi_types (rq->text, rq->text_len, types); -#ifdef USE_FRIBIDI_EX_API - fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); - max_level = fribidi_get_par_embedding_levels_ex (types, btypes, - rq->text_len, &par_type, - levels); -#else - max_level = fribidi_get_par_embedding_levels (types, rq->text_len, - &par_type, levels); -#endif - - if (par_type == FRIBIDI_PAR_LTR) - rq->resolved_dir = RAQM_DIRECTION_LTR; - else - rq->resolved_dir = RAQM_DIRECTION_RTL; - } - - if (max_level == 0) - { - ok = false; - goto done; - } - if (!_raqm_resolve_scripts (rq)) { ok = false; goto done; } - /* Get the number of bidi runs */ - runs = _raqm_reorder_runs (types, rq->text_len, par_type, levels, &run_count); + if (rq->base_dir == RAQM_DIRECTION_TTB) + { + /* Treat every thing as LTR in vertical text */ + run_count = 1; + rq->resolved_dir = RAQM_DIRECTION_TTB; + runs = malloc (sizeof (_raqm_bidi_run)); + if (runs) + { + runs->pos = 0; + runs->len = rq->text_len; + runs->level = 0; + } + } else { + runs = _raqm_bidi_itemize (rq, &run_count); + } + if (!runs) { ok = false; @@ -1197,7 +1391,7 @@ _raqm_itemize (raqm_t *rq) #ifdef RAQM_TESTING RAQM_TEST ("Number of runs before script itemization: %zu\n\n", run_count); - RAQM_TEST ("Fribidi Runs:\n"); + RAQM_TEST ("BiDi Runs:\n"); for (size_t i = 0; i < run_count; i++) { RAQM_TEST ("run[%zu]:\t start: %zu\tlength: %zu\tlevel: %d\n", @@ -1309,11 +1503,6 @@ _raqm_itemize (raqm_t *rq) done: free (runs); - free (types); -#ifdef USE_FRIBIDI_EX_API - free (btypes); -#endif - free (levels); return ok; } @@ -1328,7 +1517,7 @@ typedef struct { /* Special paired characters for script detection */ static size_t paired_len = 34; -static const FriBidiChar paired_chars[] = +static const uint32_t paired_chars[] = { 0x0028, 0x0029, /* ascii paired punctuation */ 0x003c, 0x003e, @@ -1431,7 +1620,7 @@ _raqm_stack_push (_raqm_stack_t *stack, } static int -_get_pair_index (const FriBidiChar ch) +_get_pair_index (const uint32_t ch) { int lower = 0; int upper = paired_len - 1; @@ -1569,6 +1758,7 @@ _raqm_resolve_scripts (raqm_t *rq) return true; } +#ifdef HAVE_FT_GET_TRANSFORM static void _raqm_ft_transform (int *x, int *y, @@ -1583,6 +1773,7 @@ _raqm_ft_transform (int *x, *x = vector.x; *y = vector.y; } +#endif static bool _raqm_shape (raqm_t *rq) @@ -1634,20 +1825,30 @@ _raqm_shape (raqm_t *rq) return true; } +/* Count equivalent UTF-8 bytes in codepoint */ +static size_t +_raqm_count_codepoint_utf8_bytes (uint32_t chr) +{ + if (0 == ((uint32_t) 0xffffff80 & chr)) + return 1; + else if (0 == ((uint32_t) 0xfffff800 & chr)) + return 2; + else if (0 == ((uint32_t) 0xffff0000 & chr)) + return 3; + else + return 4; +} + /* Convert index from UTF-32 to UTF-8 */ static uint32_t _raqm_u32_to_u8_index (raqm_t *rq, uint32_t index) { - FriBidiStrIndex length; - char *output = malloc ((sizeof (char) * 4 * index) + 1); + size_t length = 0; - length = fribidi_unicode_to_charset (FRIBIDI_CHAR_SET_UTF8, - rq->text, - index, - output); + for (uint32_t i = 0; i < index; ++i) + length += _raqm_count_codepoint_utf8_bytes (rq->text[i]); - free (output); return length; } @@ -1656,15 +1857,27 @@ static uint32_t _raqm_u8_to_u32_index (raqm_t *rq, uint32_t index) { - FriBidiStrIndex length; - uint32_t *output = malloc (sizeof (uint32_t) * (index + 1)); + const unsigned char *s = (const unsigned char *) rq->text_utf8; + const unsigned char *t = s; + size_t length = 0; - length = fribidi_charset_to_unicode (FRIBIDI_CHAR_SET_UTF8, - rq->text_utf8, - index, - output); + while (((size_t) (s - t) < index) && ('\0' != *s)) + { + if (0xf0 == (0xf8 & *s)) + s += 4; + else if (0xe0 == (0xf0 & *s)) + s += 3; + else if (0xc0 == (0xe0 & *s)) + s += 2; + else + s += 1; + + length++; + } + + if ((size_t) (s-t) > index) + length--; - free (output); return length; } diff --git a/src/thirdparty/raqm/raqm.h b/src/thirdparty/raqm/raqm.h index 342afc8b2..faab85591 100644 --- a/src/thirdparty/raqm/raqm.h +++ b/src/thirdparty/raqm/raqm.h @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016 Khaled Hosny + * Copyright © 2016-2021 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -156,6 +156,13 @@ RAQM_API raqm_glyph_t * raqm_get_glyphs (raqm_t *rq, size_t *length); +RAQM_API raqm_direction_t +raqm_get_par_resolved_direction (raqm_t *rq); + +RAQM_API raqm_direction_t +raqm_get_direction_at_index (raqm_t *rq, + size_t index); + RAQM_API bool raqm_index_to_position (raqm_t *rq, size_t *index, From 1b6a8c6122ad6b5e5794b04ca37660126675a8a4 Mon Sep 17 00:00:00 2001 From: DWesl <22566757+DWesl@users.noreply.github.com> Date: Sat, 18 Dec 2021 10:00:14 -0500 Subject: [PATCH 25/54] TST: Parametrize numpy roundtrip to find failing case Segfaults are annoying to debug. --- Tests/test_numpy.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Tests/test_numpy.py b/Tests/test_numpy.py index def7adf3f..936474fe8 100644 --- a/Tests/test_numpy.py +++ b/Tests/test_numpy.py @@ -189,8 +189,9 @@ def test_putdata(): assert len(im.getdata()) == len(arr) -def test_roundtrip_eye(): - for dtype in ( +@pytest.mark.parametrize( + "dtype", + ( bool, numpy.bool8, numpy.int8, @@ -202,9 +203,11 @@ def test_roundtrip_eye(): float, numpy.float32, numpy.float64, - ): - arr = numpy.eye(10, dtype=dtype) - numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) + ), +) +def test_roundtrip_eye(dtype): + arr = numpy.eye(10, dtype=dtype) + numpy.testing.assert_array_equal(arr, numpy.array(Image.fromarray(arr))) def test_zero_size(): From 7ab973c4c95494cae655825cc36c5ba9abbfabd2 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 30 Jan 2022 08:01:35 +1100 Subject: [PATCH 26/54] Updated lcms2 to 2.13 --- docs/installation.rst | 2 +- winbuild/build_prepare.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index ea0e78500..88aaf5834 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -169,7 +169,7 @@ Many of Pillow's features require external libraries: * **littlecms** provides color management * Pillow version 2.2.1 and below uses liblcms1, Pillow 2.3.0 and - above uses liblcms2. Tested with **1.19** and **2.7-2.12**. + above uses liblcms2. Tested with **1.19** and **2.7-2.13**. * **libwebp** provides the WebP format. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 3092c5a2b..78916987c 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -219,9 +219,9 @@ deps = { # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.12.tar.gz", - "filename": "lcms2-2.12.tar.gz", - "dir": "lcms2-2.12", + "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.13.tar.gz", + "filename": "lcms2-2.13.tar.gz", + "dir": "lcms2-2.13", "patch": { r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always From d8e94c206ef69989642e48aa2795a2a69f9d0577 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 30 Jan 2022 14:40:30 +1100 Subject: [PATCH 27/54] Switched from windows-2019 to windows-latest --- .github/workflows/test-mingw.yml | 2 +- .github/workflows/test-windows.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mingw.yml b/.github/workflows/test-mingw.yml index d94c7d537..8a9c1725d 100644 --- a/.github/workflows/test-mingw.yml +++ b/.github/workflows/test-mingw.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index c768838eb..c78f9fd24 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -4,7 +4,7 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: windows-2019 + runs-on: windows-latest strategy: fail-fast: false matrix: From 36f293071e76c9e0206708cf01087055f1dc0912 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 31 Jan 2022 08:05:01 +1100 Subject: [PATCH 28/54] Updated raqm to 0.9.0 --- depends/install_raqm.sh | 2 +- src/thirdparty/raqm/COPYING | 2 +- src/thirdparty/raqm/README.md | 5 +- src/thirdparty/raqm/raqm-version.h | 4 +- src/thirdparty/raqm/raqm.c | 472 ++++++++++++++++++----------- src/thirdparty/raqm/raqm.h | 11 +- 6 files changed, 311 insertions(+), 185 deletions(-) diff --git a/depends/install_raqm.sh b/depends/install_raqm.sh index 39b998d05..992503650 100755 --- a/depends/install_raqm.sh +++ b/depends/install_raqm.sh @@ -2,7 +2,7 @@ # install raqm -archive=libraqm-0.8.0 +archive=libraqm-0.9.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz diff --git a/src/thirdparty/raqm/COPYING b/src/thirdparty/raqm/COPYING index 78db981bb..c605a5dc6 100644 --- a/src/thirdparty/raqm/COPYING +++ b/src/thirdparty/raqm/COPYING @@ -1,7 +1,7 @@ The MIT License (MIT) Copyright © 2015 Information Technology Authority (ITA) -Copyright © 2016-2021 Khaled Hosny +Copyright © 2016-2022 Khaled Hosny Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/src/thirdparty/raqm/README.md b/src/thirdparty/raqm/README.md index 17ce2efc9..02e996e7a 100644 --- a/src/thirdparty/raqm/README.md +++ b/src/thirdparty/raqm/README.md @@ -68,6 +68,7 @@ Projects using Raqm 3. [FontView](https://github.com/googlei18n/fontview) 4. [Pillow](https://github.com/python-pillow) 5. [mplcairo](https://github.com/anntzer/mplcairo) +6. [CEGUI](https://github.com/cegui/cegui) The following projects have patches to support complex text layout using Raqm: @@ -77,8 +78,8 @@ The following projects have patches to support complex text layout using Raqm: -[1]: http://fribidi.org +[1]: https://github.com/fribidi/fribidi [2]: https://github.com/Tehreer/SheenBidi -[3]: http://harfbuzz.org +[3]: https://github.com/harfbuzz/harfbuzz [4]: https://www.freetype.org [5]: https://www.gtk.org/gtk-doc diff --git a/src/thirdparty/raqm/raqm-version.h b/src/thirdparty/raqm/raqm-version.h index e96b1aea5..78b70a561 100644 --- a/src/thirdparty/raqm/raqm-version.h +++ b/src/thirdparty/raqm/raqm-version.h @@ -32,10 +32,10 @@ #define _RAQM_VERSION_H_ #define RAQM_VERSION_MAJOR 0 -#define RAQM_VERSION_MINOR 8 +#define RAQM_VERSION_MINOR 9 #define RAQM_VERSION_MICRO 0 -#define RAQM_VERSION_STRING "0.8.0" +#define RAQM_VERSION_STRING "0.9.0" #define RAQM_VERSION_ATLEAST(major,minor,micro) \ ((major)*10000+(minor)*100+(micro) <= \ diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index 46890b9a7..f852542b4 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016-2021 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -24,7 +24,6 @@ #ifdef HAVE_CONFIG_H #include "config.h" -#undef HAVE_CONFIG_H // Workaround for Fribidi 1.0.5 and earlier #endif #include @@ -38,9 +37,6 @@ #else #include "../fribidi-shim/fribidi.h" #endif -#if FRIBIDI_MAJOR_VERSION >= 1 -#define USE_FRIBIDI_EX_API -#endif #endif #include @@ -190,13 +186,9 @@ typedef FriBidiLevel _raqm_bidi_level_t; #endif -typedef enum { - RAQM_FLAG_NONE = 0, - RAQM_FLAG_UTF8 = 1 << 0 -} _raqm_flags_t; - typedef struct { FT_Face ftface; + int ftloadflags; hb_language_t lang; hb_script_t script; } _raqm_text_info; @@ -209,6 +201,7 @@ struct _raqm { uint32_t *text; char *text_utf8; size_t text_len; + size_t text_capacity_bytes; _raqm_text_info *text_info; @@ -219,17 +212,17 @@ struct _raqm { size_t features_len; raqm_run_t *runs; + raqm_run_t *runs_pool; + raqm_glyph_t *glyphs; + size_t glyphs_capacity; - _raqm_flags_t flags; - - int ft_loadflags; int invisible_glyph; }; struct _raqm_run { - int pos; - int len; + uint32_t pos; + uint32_t len; hb_direction_t direction; hb_script_t script; @@ -243,31 +236,21 @@ static uint32_t _raqm_u8_to_u32_index (raqm_t *rq, uint32_t index); -static bool +static void _raqm_init_text_info (raqm_t *rq) { - hb_language_t default_lang; - - if (rq->text_info) - return true; - - rq->text_info = malloc (sizeof (_raqm_text_info) * rq->text_len); - if (!rq->text_info) - return false; - - default_lang = hb_language_get_default (); + hb_language_t default_lang = hb_language_get_default (); for (size_t i = 0; i < rq->text_len; i++) { rq->text_info[i].ftface = NULL; + rq->text_info[i].ftloadflags = -1; rq->text_info[i].lang = default_lang; rq->text_info[i].script = HB_SCRIPT_INVALID; } - - return true; } static void -_raqm_free_text_info (raqm_t *rq) +_raqm_release_text_info (raqm_t *rq) { if (!rq->text_info) return; @@ -277,9 +260,6 @@ _raqm_free_text_info (raqm_t *rq) if (rq->text_info[i].ftface) FT_Done_Face (rq->text_info[i].ftface); } - - free (rq->text_info); - rq->text_info = NULL; } static bool @@ -289,6 +269,9 @@ _raqm_compare_text_info (_raqm_text_info a, if (a.ftface != b.ftface) return false; + if (a.ftloadflags != b.ftloadflags) + return false; + if (a.lang != b.lang) return false; @@ -298,6 +281,88 @@ _raqm_compare_text_info (_raqm_text_info a, return true; } +static void +_raqm_free_text(raqm_t* rq) +{ + free (rq->text); + rq->text = NULL; + rq->text_info = NULL; + rq->text_utf8 = NULL; + rq->text_len = 0; + rq->text_capacity_bytes = 0; +} + +static bool +_raqm_alloc_text(raqm_t *rq, + size_t len, + bool need_utf8) +{ + /* Allocate contiguous memory block for texts and text_info */ + size_t mem_size = (sizeof (uint32_t) + sizeof (_raqm_text_info)) * len; + if (need_utf8) + mem_size += sizeof (char) * len; + + if (mem_size > rq->text_capacity_bytes) + { + void* new_mem = realloc (rq->text, mem_size); + if (!new_mem) + { + _raqm_free_text (rq); + return false; + } + + rq->text_capacity_bytes = mem_size; + rq->text = new_mem; + } + + rq->text_info = (_raqm_text_info*)(rq->text + len); + rq->text_utf8 = need_utf8 ? (char*)(rq->text_info + len) : NULL; + + return true; +} + +static raqm_run_t* +_raqm_alloc_run (raqm_t *rq) +{ + raqm_run_t *run = rq->runs_pool; + if (run) + { + rq->runs_pool = run->next; + } + else + { + run = malloc (sizeof (raqm_run_t)); + run->font = NULL; + run->buffer = NULL; + } + + run->pos = 0; + run->len = 0; + run->direction = HB_DIRECTION_INVALID; + run->script = HB_SCRIPT_INVALID; + run->next = NULL; + + return run; +} + +static void +_raqm_free_runs (raqm_run_t *runs) +{ + while (runs) + { + raqm_run_t *run = runs; + runs = runs->next; + + if (run->buffer) + hb_buffer_destroy (run->buffer); + + if (run->font) + hb_font_destroy (run->font); + + free (run); + } +} + /** * raqm_create: * @@ -322,26 +387,26 @@ raqm_create (void) rq->ref_count = 1; - rq->text = NULL; - rq->text_utf8 = NULL; - rq->text_len = 0; - - rq->text_info = NULL; - rq->base_dir = RAQM_DIRECTION_DEFAULT; rq->resolved_dir = RAQM_DIRECTION_DEFAULT; rq->features = NULL; rq->features_len = 0; - rq->runs = NULL; - rq->glyphs = NULL; - - rq->flags = RAQM_FLAG_NONE; - - rq->ft_loadflags = -1; rq->invisible_glyph = 0; + rq->text = NULL; + rq->text_utf8 = NULL; + rq->text_info = NULL; + rq->text_capacity_bytes = 0; + rq->text_len = 0; + + rq->runs = NULL; + rq->runs_pool = NULL; + + rq->glyphs = NULL; + rq->glyphs_capacity = 0; + return rq; } @@ -366,28 +431,13 @@ raqm_reference (raqm_t *rq) return rq; } -static void -_raqm_free_runs (raqm_t *rq) -{ - raqm_run_t *runs = rq->runs; - while (runs) - { - raqm_run_t *run = runs; - runs = runs->next; - - hb_buffer_destroy (run->buffer); - hb_font_destroy (run->font); - free (run); - } -} - /** * raqm_destroy: * @rq: a #raqm_t. * * Decreases the reference count on @rq by one. If the result is zero, then @rq * and all associated resources are freed. - * See cairo_reference(). + * See raqm_reference(). * * Since: 0.1 */ @@ -397,14 +447,60 @@ raqm_destroy (raqm_t *rq) if (!rq || --rq->ref_count != 0) return; - free (rq->text); - free (rq->text_utf8); - _raqm_free_text_info (rq); - _raqm_free_runs (rq); + _raqm_release_text_info (rq); + _raqm_free_text (rq); + _raqm_free_runs (rq->runs); + _raqm_free_runs (rq->runs_pool); free (rq->glyphs); + free (rq->features); free (rq); } +/** + * raqm_clear_contents: + * @rq: a #raqm_t. + * + * Clears internal state of previously used raqm_t object, making it ready + * for reuse and keeping some of allocated memory to increase performance. + * + * Since: 0.9 + */ +void +raqm_clear_contents (raqm_t *rq) +{ + if (!rq) + return; + + _raqm_release_text_info (rq); + + /* Return allocated runs to the pool, keep hb buffers for reuse */ + raqm_run_t *run = rq->runs; + while (run) + { + if (run->buffer) + hb_buffer_reset (run->buffer); + + if (run->font) + { + hb_font_destroy (run->font); + run->font = NULL; + } + + if (!run->next) + { + run->next = rq->runs_pool; + rq->runs_pool = rq->runs; + rq->runs = NULL; + break; + } + + run = run->next; + } + + rq->text_len = 0; + rq->resolved_dir = RAQM_DIRECTION_DEFAULT; +} + /** * raqm_set_text: * @rq: a #raqm_t. @@ -429,23 +525,20 @@ raqm_set_text (raqm_t *rq, if (!rq || !text) return false; - rq->text_len = len; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; /* Empty string, don’t fail but do nothing */ if (!len) return true; - free (rq->text); + if (!_raqm_alloc_text(rq, len, false)) + return false; - rq->text = malloc (sizeof (uint32_t) * rq->text_len); - if (!rq->text) - return false; - - _raqm_free_text_info (rq); - if (!_raqm_init_text_info (rq)) - return false; - - memcpy (rq->text, text, sizeof (uint32_t) * rq->text_len); + rq->text_len = len; + memcpy (rq->text, text, sizeof (uint32_t) * len); + _raqm_init_text_info (rq); return true; } @@ -511,41 +604,29 @@ _raqm_u8_to_u32 (const char *text, size_t len, uint32_t *unicode) * Since: 0.1 */ bool -raqm_set_text_utf8 (raqm_t *rq, - const char *text, - size_t len) +raqm_set_text_utf8 (raqm_t *rq, + const char *text, + size_t len) { - uint32_t *unicode; - size_t ulen; - bool ok; - if (!rq || !text) return false; + /* Call raqm_clear_contents to reuse this raqm_t */ + if (rq->text_len) + return false; + /* Empty string, don’t fail but do nothing */ if (!len) - { - rq->text_len = len; return true; - } - rq->flags |= RAQM_FLAG_UTF8; - - rq->text_utf8 = malloc (sizeof (char) * len); - if (!rq->text_utf8) - return false; - - unicode = malloc (sizeof (uint32_t) * len); - if (!unicode) - return false; + if (!_raqm_alloc_text(rq, len, true)) + return false; + rq->text_len = _raqm_u8_to_u32 (text, len, rq->text); memcpy (rq->text_utf8, text, sizeof (char) * len); + _raqm_init_text_info (rq); - ulen = _raqm_u8_to_u32 (text, len, unicode); - ok = raqm_set_text (rq, unicode, ulen); - - free (unicode); - return ok; + return true; } /** @@ -561,7 +642,7 @@ raqm_set_text_utf8 (raqm_t *rq, * * The default is #RAQM_DIRECTION_DEFAULT, which determines the paragraph * direction based on the first character with strong bidi type (see [rule - * P2](http://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), + * P2](https://unicode.org/reports/tr9/#P2) in Unicode Bidirectional Algorithm), * which can be good enough for many cases but has problems when a mainly * right-to-left paragraph starts with a left-to-right character and vice versa * as the detected paragraph direction will be the wrong one, or when text does @@ -629,7 +710,7 @@ raqm_set_language (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -686,13 +767,14 @@ raqm_add_font_feature (raqm_t *rq, ok = hb_feature_from_string (feature, len, &fea); if (ok) { - rq->features_len++; - rq->features = realloc (rq->features, - sizeof (hb_feature_t) * (rq->features_len)); - if (!rq->features) + void* new_features = realloc (rq->features, + sizeof (hb_feature_t) * (rq->features_len + 1)); + if (!new_features) return false; - rq->features[rq->features_len - 1] = fea; + rq->features = new_features; + rq->features[rq->features_len] = fea; + rq->features_len++; } return ok; @@ -700,12 +782,13 @@ raqm_add_font_feature (raqm_t *rq, static hb_font_t * _raqm_create_hb_font (raqm_t *rq, - FT_Face face) + FT_Face face, + int loadflags) { hb_font_t *font = hb_ft_font_create_referenced (face); - if (rq->ft_loadflags >= 0) - hb_ft_font_set_load_flags (font, rq->ft_loadflags); + if (loadflags >= 0) + hb_ft_font_set_load_flags (font, loadflags); return font; } @@ -796,7 +879,7 @@ raqm_set_freetype_face_range (raqm_t *rq, if (!rq->text_len) return true; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { start = _raqm_u8_to_u32_index (rq, start); end = _raqm_u8_to_u32_index (rq, end); @@ -805,6 +888,30 @@ raqm_set_freetype_face_range (raqm_t *rq, return _raqm_set_freetype_face (rq, face, start, end); } +static bool +_raqm_set_freetype_load_flags (raqm_t *rq, + int flags, + size_t start, + size_t end) +{ + if (!rq) + return false; + + if (!rq->text_len) + return true; + + if (start >= rq->text_len || end > rq->text_len) + return false; + + if (!rq->text_info) + return false; + + for (size_t i = start; i < end; i++) + rq->text_info[i].ftloadflags = flags; + + return true; +} + /** * raqm_set_freetype_load_flags: * @rq: a #raqm_t. @@ -823,14 +930,59 @@ raqm_set_freetype_face_range (raqm_t *rq, */ bool raqm_set_freetype_load_flags (raqm_t *rq, - int flags) + int flags) { + return _raqm_set_freetype_load_flags(rq, flags, 0, rq->text_len); +} + +/** + * raqm_set_freetype_load_flags_range: + * @rq: a #raqm_t. + * @flags: FreeType load flags. + * @start: index of first character that should use @flags. + * @len: number of characters using @flags. + * + * Sets the load flags passed to FreeType when loading glyphs for @len-number + * of characters staring at @start. Flags should be the same as used by the + * client when rendering corresponding FreeType glyphs. The @start and @len + * are input string array indices (i.e. counting bytes in UTF-8 and scaler + * values in UTF-32). + * + * This method can be used repeatedly to set different flags for different + * parts of the text. It is the responsibility of the client to make sure that + * flag ranges cover the whole text. + * + * This requires version of HarfBuzz that has hb_ft_font_set_load_flags(), for + * older version the flags will be ignored. + * + * See also raqm_set_freetype_load_flags(). + * + * Return value: + * %true if no errors happened, %false otherwise. + * + * Since: 0.9 + */ +bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len) +{ + size_t end = start + len; + if (!rq) return false; - rq->ft_loadflags = flags; + if (!rq->text_len) + return true; - return true; + if (rq->text_utf8) + { + start = _raqm_u8_to_u32_index (rq, start); + end = _raqm_u8_to_u32_index (rq, end); + } + + return _raqm_set_freetype_load_flags (rq, flags, start, end); } /** @@ -841,17 +993,10 @@ raqm_set_freetype_load_flags (raqm_t *rq, * Sets the glyph id to be used for invisible glyhphs. * * If @gid is negative, invisible glyphs will be suppressed from the output. - * This requires HarfBuzz 1.8.0 or later. If raqm is used with an earlier - * HarfBuzz version, the return value will be %false and the shaping behavior - * does not change. * * If @gid is zero, invisible glyphs will be rendered as space. - * This works on all versions of HarfBuzz. * * If @gid is a positive number, it will be used for invisible glyphs. - * This requires a version of HarfBuzz that has - * hb_buffer_set_invisible_glyph(). For older versions, the return value - * will be %false and the shaping behavior does not change. * * Return value: * %true if no errors happened, %false otherwise. @@ -865,17 +1010,6 @@ raqm_set_invisible_glyph (raqm_t *rq, if (!rq) return false; -#ifndef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH - if (gid > 0) - return false; -#endif - -#if !defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) || \ - !HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES - if (gid < 0) - return false; -#endif - rq->invisible_glyph = gid; return true; } @@ -961,18 +1095,21 @@ raqm_get_glyphs (raqm_t *rq, for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) count += hb_buffer_get_length (run->buffer); - *length = count; - - if (rq->glyphs) - free (rq->glyphs); - - rq->glyphs = malloc (sizeof (raqm_glyph_t) * count); - if (!rq->glyphs) + if (count > rq->glyphs_capacity) { - *length = 0; - return NULL; + void* new_mem = realloc (rq->glyphs, sizeof (raqm_glyph_t) * count); + if (!new_mem) + { + *length = 0; + return NULL; + } + + rq->glyphs = new_mem; + rq->glyphs_capacity = count; } + *length = count; + RAQM_TEST ("Glyph information:\n"); count = 0; @@ -1005,7 +1142,7 @@ raqm_get_glyphs (raqm_t *rq, count += len; } - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) { #ifdef RAQM_TESTING RAQM_TEST ("\nUTF-32 clusters:"); @@ -1276,24 +1413,14 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) FriBidiCharType *types; _raqm_bidi_level_t *levels; int max_level = 0; -#ifdef USE_FRIBIDI_EX_API FriBidiBracketType *btypes; -#endif types = calloc (rq->text_len, sizeof (FriBidiCharType)); -#ifdef USE_FRIBIDI_EX_API btypes = calloc (rq->text_len, sizeof (FriBidiBracketType)); -#endif levels = calloc (rq->text_len, sizeof (_raqm_bidi_level_t)); - if (!types || !levels -#ifdef USE_FRIBIDI_EX_API - || !btypes -#endif - ) - { + if (!types || !levels || !btypes) goto done; - } if (rq->base_dir == RAQM_DIRECTION_RTL) par_type = FRIBIDI_PAR_RTL; @@ -1301,15 +1428,10 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) par_type = FRIBIDI_PAR_LTR; fribidi_get_bidi_types (rq->text, rq->text_len, types); -#ifdef USE_FRIBIDI_EX_API fribidi_get_bracket_types (rq->text, rq->text_len, types, btypes); max_level = fribidi_get_par_embedding_levels_ex (types, btypes, rq->text_len, &par_type, levels); -#else - max_level = fribidi_get_par_embedding_levels (types, rq->text_len, - &par_type, levels); -#endif if (par_type == FRIBIDI_PAR_LTR) rq->resolved_dir = RAQM_DIRECTION_LTR; @@ -1325,9 +1447,7 @@ _raqm_bidi_itemize (raqm_t *rq, size_t *run_count) done: free (types); free (levels); -#ifdef USE_FRIBIDI_EX_API free (btypes); -#endif return runs; } @@ -1403,7 +1523,7 @@ _raqm_itemize (raqm_t *rq) last = NULL; for (size_t i = 0; i < run_count; i++) { - raqm_run_t *run = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *run = _raqm_alloc_run (rq); if (!run) { ok = false; @@ -1422,13 +1542,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos + runs[i].len - 1; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (int j = runs[i].len - 1; j >= 0; j--) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1438,7 +1559,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1453,13 +1575,14 @@ _raqm_itemize (raqm_t *rq) { run->pos = runs[i].pos; run->script = rq->text_info[run->pos].script; - run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface); + run->font = _raqm_create_hb_font (rq, rq->text_info[run->pos].ftface, + rq->text_info[run->pos].ftloadflags); for (size_t j = 0; j < runs[i].len; j++) { _raqm_text_info info = rq->text_info[runs[i].pos + j]; if (!_raqm_compare_text_info (rq->text_info[run->pos], info)) { - raqm_run_t *newrun = calloc (1, sizeof (raqm_run_t)); + raqm_run_t *newrun = _raqm_alloc_run (rq); if (!newrun) { ok = false; @@ -1469,7 +1592,8 @@ _raqm_itemize (raqm_t *rq) newrun->len = 1; newrun->direction = _raqm_hb_dir (rq, runs[i].level); newrun->script = info.script; - newrun->font = _raqm_create_hb_font (rq, info.ftface); + newrun->font = _raqm_create_hb_font (rq, info.ftface, + info.ftloadflags); run->next = newrun; run = newrun; } @@ -1758,7 +1882,6 @@ _raqm_resolve_scripts (raqm_t *rq) return true; } -#ifdef HAVE_FT_GET_TRANSFORM static void _raqm_ft_transform (int *x, int *y, @@ -1773,22 +1896,19 @@ _raqm_ft_transform (int *x, *x = vector.x; *y = vector.y; } -#endif static bool _raqm_shape (raqm_t *rq) { hb_buffer_flags_t hb_buffer_flags = HB_BUFFER_FLAG_BOT | HB_BUFFER_FLAG_EOT; -#if defined(HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES) && \ - HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES if (rq->invisible_glyph < 0) hb_buffer_flags |= HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES; -#endif for (raqm_run_t *run = rq->runs; run != NULL; run = run->next) { - run->buffer = hb_buffer_create (); + if (!run->buffer) + run->buffer = hb_buffer_create (); hb_buffer_add_utf32 (run->buffer, rq->text, rq->text_len, run->pos, run->len); @@ -1797,15 +1917,12 @@ _raqm_shape (raqm_t *rq) hb_buffer_set_direction (run->buffer, run->direction); hb_buffer_set_flags (run->buffer, hb_buffer_flags); -#ifdef HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH if (rq->invisible_glyph > 0) hb_buffer_set_invisible_glyph (run->buffer, rq->invisible_glyph); -#endif hb_shape_full (run->font, run->buffer, rq->features, rq->features_len, NULL); -#ifdef HAVE_FT_GET_TRANSFORM { FT_Matrix matrix; hb_glyph_position_t *pos; @@ -1819,7 +1936,6 @@ _raqm_shape (raqm_t *rq) _raqm_ft_transform (&pos[i].x_offset, &pos[i].y_offset, matrix); } } -#endif } return true; @@ -1917,7 +2033,7 @@ raqm_index_to_position (raqm_t *rq, if (rq == NULL) return false; - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u8_to_u32_index (rq, *index); if (*index >= rq->text_len) @@ -1974,7 +2090,7 @@ raqm_index_to_position (raqm_t *rq, } found: - if (rq->flags & RAQM_FLAG_UTF8) + if (rq->text_utf8) *index = _raqm_u32_to_u8_index (rq, *index); RAQM_TEST ("The position is %d at index %zu\n",*x ,*index); return true; diff --git a/src/thirdparty/raqm/raqm.h b/src/thirdparty/raqm/raqm.h index faab85591..bdb5a50d8 100644 --- a/src/thirdparty/raqm/raqm.h +++ b/src/thirdparty/raqm/raqm.h @@ -1,6 +1,6 @@ /* * Copyright © 2015 Information Technology Authority (ITA) - * Copyright © 2016-2021 Khaled Hosny + * Copyright © 2016-2022 Khaled Hosny * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to @@ -106,6 +106,9 @@ raqm_reference (raqm_t *rq); RAQM_API void raqm_destroy (raqm_t *rq); +RAQM_API void +raqm_clear_contents (raqm_t *rq); + RAQM_API bool raqm_set_text (raqm_t *rq, const uint32_t *text, @@ -145,6 +148,12 @@ RAQM_API bool raqm_set_freetype_load_flags (raqm_t *rq, int flags); +RAQM_API bool +raqm_set_freetype_load_flags_range (raqm_t *rq, + int flags, + size_t start, + size_t len); + RAQM_API bool raqm_set_invisible_glyph (raqm_t *rq, int gid); From 999ef3e4eaf793612d9f3525fcbff2bac629e135 Mon Sep 17 00:00:00 2001 From: nulano Date: Mon, 31 Jan 2022 00:11:47 +0000 Subject: [PATCH 29/54] Revert "detect FreeType / HarfBuzz features" This reverts commit 6565d13275cead21dc6f369204f0dc3d0b43bc18. --- src/thirdparty/raqm/raqm.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/thirdparty/raqm/raqm.c b/src/thirdparty/raqm/raqm.c index f852542b4..13f6e1f02 100644 --- a/src/thirdparty/raqm/raqm.c +++ b/src/thirdparty/raqm/raqm.c @@ -42,21 +42,6 @@ #include #include -#if FREETYPE_MAJOR > 2 || \ - FREETYPE_MAJOR == 2 && FREETYPE_MINOR >= 11 -#define HAVE_FT_GET_TRANSFORM -#endif - -#if HB_VERSION_ATLEAST(2, 0, 0) -#define HAVE_HB_BUFFER_SET_INVISIBLE_GLYPH -#endif - -#if HB_VERSION_ATLEAST(1, 8, 0) -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 1 -#else -#define HAVE_DECL_HB_BUFFER_FLAG_REMOVE_DEFAULT_IGNORABLES 0 -#endif - #include "raqm.h" /** From 3ac2da533a1e74f4a1bb38e8e812190c49fbdc53 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 18:56:52 +0200 Subject: [PATCH 30/54] Fix lcms2 URL --- winbuild/build_prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 78916987c..182146fbc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -219,7 +219,7 @@ deps = { # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.12/lcms2-2.13.tar.gz", + "url": SF_MIRROR + "/project/lcms/lcms/2.13/lcms2-2.13.tar.gz", "filename": "lcms2-2.13.tar.gz", "dir": "lcms2-2.13", "patch": { From cc4f9a2a7dcb3e69dd4ca4ab752ebcd0fba0bbb2 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 20:38:47 +0200 Subject: [PATCH 31/54] Update harfbuzz to 3.3.0 --- winbuild/build_prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 3092c5a2b..e6bcbc6fc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -278,9 +278,9 @@ deps = { "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.2.0.zip", - "filename": "harfbuzz-3.2.0.zip", - "dir": "harfbuzz-3.2.0", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.0.zip", + "filename": "harfbuzz-3.3.0.zip", + "dir": "harfbuzz-3.3.0", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), From 87ab18f804ed860f4315038980bcf20044ae352c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 23:27:35 +0200 Subject: [PATCH 32/54] Update harfbuzz to 3.3.1 --- winbuild/build_prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index e6bcbc6fc..220956adc 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -278,9 +278,9 @@ deps = { "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.0.zip", - "filename": "harfbuzz-3.3.0.zip", - "dir": "harfbuzz-3.3.0", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.1.zip", + "filename": "harfbuzz-3.3.1.zip", + "dir": "harfbuzz-3.3.1", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), From 54e9decde30915e1f89ed7cda0bc49870707a02e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 31 Jan 2022 23:41:16 +0200 Subject: [PATCH 33/54] Remove EOL CentOS 8 --- .github/workflows/test-docker.yml | 1 - docs/installation.rst | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index df04d0a6c..656df5e91 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -19,7 +19,6 @@ jobs: amazon-2-amd64, arch, centos-7-amd64, - centos-8-amd64, centos-stream-8-amd64, debian-10-buster-x86, debian-11-bullseye-x86, diff --git a/docs/installation.rst b/docs/installation.rst index 88aaf5834..984a689c2 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -453,8 +453,6 @@ These platforms are built and tested for every change. +----------------------------------+----------------------------+---------------------+ | CentOS 7 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ -| CentOS 8 | 3.9 | x86-64 | -+----------------------------------+----------------------------+---------------------+ | CentOS Stream 8 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ | Debian 10 Buster | 3.7 | x86 | @@ -530,6 +528,8 @@ These platforms have been reported to work at the versions mentioned. +----------------------------------+---------------------------+------------------+--------------+ | CentOS 6.3 | 2.7, 3.3 | |x86 | +----------------------------------+---------------------------+------------------+--------------+ +| CentOS 8 | 3.9 | 9.0.0 |x86-64 | ++----------------------------------+---------------------------+------------------+--------------+ | Fedora 23 | 2.7, 3.4 | 3.1.0 |x86-64 | +----------------------------------+---------------------------+------------------+--------------+ | Ubuntu Linux 12.04 LTS (Precise) | | 2.6, 3.2, 3.3, 3.4, 3.5 | 3.4.1 |x86,x86-64 | From f22d3561fd764e9600fe4640856c50ad74a90eaf Mon Sep 17 00:00:00 2001 From: Gabor Kertesz Date: Fri, 29 Oct 2021 12:31:40 +0200 Subject: [PATCH 34/54] Windows: Enable ARM64 for MSVC This patch enables ARM64 as a new platform for Windows. Platform query and documentation is updated accordingly. --- winbuild/build.rst | 6 +++--- winbuild/build_prepare.py | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/winbuild/build.rst b/winbuild/build.rst index b30a94226..661c5a5ec 100644 --- a/winbuild/build.rst +++ b/winbuild/build.rst @@ -24,7 +24,7 @@ Download and install: * `CMake 3.12 or newer `_ (also available as Visual Studio component C++ CMake tools for Windows) -* `NASM `_ +* x86/x64: `NASM `_ Any version of Visual Studio 2017 or newer should be supported, including Visual Studio 2017 Community, or Build Tools for Visual Studio 2019. @@ -42,8 +42,8 @@ behaviour of ``build_prepare.py``: If ``PYTHON`` is unset, the version of Python used to run ``build_prepare.py`` will be used. If only ``PYTHON`` is set, ``EXECUTABLE`` defaults to ``python.exe``. -* ``ARCHITECTURE`` is used to select a ``x86`` or ``x64`` build. By default, - uses same architecture as the version of Python used to run ``build_prepare.py``. +* ``ARCHITECTURE`` is used to select a ``x86``, ``x64`` or ``ARM64``build. + By default, uses same architecture as the version of Python used to run ``build_prepare.py``. is used. * ``PILLOW_BUILD`` can be used to override the ``winbuild\build`` directory path, used to store generated build scripts and compiled libraries. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 4d9f79211..bebafff9f 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -1,4 +1,5 @@ import os +import platform import shutil import struct import subprocess @@ -93,6 +94,7 @@ SF_MIRROR = "http://iweb.dl.sourceforge.net" architectures = { "x86": {"vcvars_arch": "x86", "msbuild_arch": "Win32"}, "x64": {"vcvars_arch": "x86_amd64", "msbuild_arch": "x64"}, + "ARM64": {"vcvars_arch": "x86_arm64", "msbuild_arch": "ARM64"}, } header = [ @@ -490,7 +492,10 @@ if __name__ == "__main__": python_dir = os.environ.get("PYTHON") python_exe = os.environ.get("EXECUTABLE", "python.exe") architecture = os.environ.get( - "ARCHITECTURE", "x86" if struct.calcsize("P") == 4 else "x64" + "ARCHITECTURE", + "ARM64" + if platform.machine() == "ARM64" + else ("x86" if struct.calcsize("P") == 4 else "x64"), ) build_dir = os.environ.get("PILLOW_BUILD", os.path.join(winbuild_dir, "build")) sources_dir = "" From 22a1f6db5e097783885be99970ae45496fc296e8 Mon Sep 17 00:00:00 2001 From: Gabor Kertesz Date: Fri, 29 Oct 2021 12:20:56 +0200 Subject: [PATCH 35/54] lcms2: Update to VS2019 In order to enable win-arm64, VS2019 should be used, while other platforms should work with newer version as well. Tested on x64-win10. --- winbuild/build_prepare.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index bebafff9f..01f1bac29 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -225,21 +225,21 @@ deps = { "filename": "lcms2-2.13.tar.gz", "dir": "lcms2-2.13", "patch": { - r"Projects\VC2017\lcms2_static\lcms2_static.vcxproj": { + r"Projects\VC2019\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always "MultiThreaded": "MultiThreadedDLL", # noqa: E501 # retarget to default toolset (selected by vcvarsall.bat) - "v141": "$(DefaultPlatformToolset)", # noqa: E501 + "v142": "$(DefaultPlatformToolset)", # noqa: E501 # retarget to latest (selected by vcvarsall.bat) - "10.0.17134.0": "$(WindowsSDKVersion)", # noqa: E501 + "10.0": "$(WindowsSDKVersion)", # noqa: E501 } }, "build": [ cmd_rmdir("Lib"), - cmd_rmdir(r"Projects\VC2017\Release"), - cmd_msbuild(r"Projects\VC2017\lcms2.sln", "Release", "Clean"), + cmd_rmdir(r"Projects\VC2019\Release"), + cmd_msbuild(r"Projects\VC2019\lcms2.sln", "Release", "Clean"), cmd_msbuild( - r"Projects\VC2017\lcms2.sln", "Release", "lcms2_static:Rebuild" + r"Projects\VC2019\lcms2.sln", "Release", "lcms2_static:Rebuild" ), cmd_xcopy("include", "{inc_dir}"), ], From fb7edfda68a8972ec487eab845506fc08f25cae8 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 2 Feb 2022 11:49:31 +1100 Subject: [PATCH 36/54] Improved consistency of returning an image access object from load() --- Tests/test_file_eps.py | 9 +++++++++ Tests/test_file_gbr.py | 22 +++++++++++++++------- Tests/test_file_icns.py | 8 ++++++++ Tests/test_file_ico.py | 5 +++++ Tests/test_file_wal.py | 16 ++++++++++------ Tests/test_file_wmf.py | 6 ++++++ src/PIL/EpsImagePlugin.py | 12 ++++++------ src/PIL/GbrImagePlugin.py | 10 ++++------ src/PIL/IcnsImagePlugin.py | 9 +++++---- src/PIL/IcoImagePlugin.py | 2 +- src/PIL/ImageFile.py | 1 + src/PIL/WalImageFile.py | 13 +++++-------- src/PIL/WmfImagePlugin.py | 2 +- 13 files changed, 76 insertions(+), 39 deletions(-) diff --git a/Tests/test_file_eps.py b/Tests/test_file_eps.py index 4c0b96f73..1790f4f77 100644 --- a/Tests/test_file_eps.py +++ b/Tests/test_file_eps.py @@ -58,6 +58,15 @@ def test_sanity(): assert image2_scale2.format == "EPS" +@pytest.mark.skipif(not HAS_GHOSTSCRIPT, reason="Ghostscript not available") +def test_load(): + with Image.open(FILE1) as im: + assert im.load()[0, 0] == (255, 255, 255) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (255, 255, 255) + + def test_invalid_file(): invalid_file = "Tests/images/flower.jpg" diff --git a/Tests/test_file_gbr.py b/Tests/test_file_gbr.py index 8d7fcf147..1ea8af8ee 100644 --- a/Tests/test_file_gbr.py +++ b/Tests/test_file_gbr.py @@ -5,20 +5,28 @@ from PIL import GbrImagePlugin, Image from .helper import assert_image_equal_tofile -def test_invalid_file(): - invalid_file = "Tests/images/flower.jpg" - - with pytest.raises(SyntaxError): - GbrImagePlugin.GbrImageFile(invalid_file) - - def test_gbr_file(): with Image.open("Tests/images/gbr.gbr") as im: assert_image_equal_tofile(im, "Tests/images/gbr.png") +def test_load(): + with Image.open("Tests/images/gbr.gbr") as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_multiple_load_operations(): with Image.open("Tests/images/gbr.gbr") as im: im.load() im.load() assert_image_equal_tofile(im, "Tests/images/gbr.png") + + +def test_invalid_file(): + invalid_file = "Tests/images/flower.jpg" + + with pytest.raises(SyntaxError): + GbrImagePlugin.GbrImageFile(invalid_file) diff --git a/Tests/test_file_icns.py b/Tests/test_file_icns.py index 3afbbeaac..b492f6cb2 100644 --- a/Tests/test_file_icns.py +++ b/Tests/test_file_icns.py @@ -28,6 +28,14 @@ def test_sanity(): assert im.format == "ICNS" +def test_load(): + with Image.open(TEST_FILE) as im: + assert im.load()[0, 0] == (0, 0, 0, 0) + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == (0, 0, 0, 0) + + def test_save(tmp_path): temp_file = str(tmp_path / "temp.icns") diff --git a/Tests/test_file_ico.py b/Tests/test_file_ico.py index 317264db6..73ac6f742 100644 --- a/Tests/test_file_ico.py +++ b/Tests/test_file_ico.py @@ -18,6 +18,11 @@ def test_sanity(): assert im.get_format_mimetype() == "image/x-icon" +def test_load(): + with Image.open(TEST_ICO_FILE) as im: + assert im.load()[0, 0] == (1, 1, 9, 255) + + def test_mask(): with Image.open("Tests/images/hopper_mask.ico") as im: assert_image_equal_tofile(im, "Tests/images/hopper_mask.png") diff --git a/Tests/test_file_wal.py b/Tests/test_file_wal.py index f25b42fe0..4be46e9d6 100644 --- a/Tests/test_file_wal.py +++ b/Tests/test_file_wal.py @@ -2,15 +2,11 @@ from PIL import WalImageFile from .helper import assert_image_equal_tofile +TEST_FILE = "Tests/images/hopper.wal" + def test_open(): - # Arrange - TEST_FILE = "Tests/images/hopper.wal" - - # Act with WalImageFile.open(TEST_FILE) as im: - - # Assert assert im.format == "WAL" assert im.format_description == "Quake2 Texture" assert im.mode == "P" @@ -19,3 +15,11 @@ def test_open(): assert isinstance(im, WalImageFile.WalImageFile) assert_image_equal_tofile(im, "Tests/images/hopper_wal.png") + + +def test_load(): + with WalImageFile.open(TEST_FILE) as im: + assert im.load()[0, 0] == 122 + + # Test again now that it has already been loaded once + assert im.load()[0, 0] == 122 diff --git a/Tests/test_file_wmf.py b/Tests/test_file_wmf.py index 3f8bc96cc..d6769a24b 100644 --- a/Tests/test_file_wmf.py +++ b/Tests/test_file_wmf.py @@ -24,6 +24,12 @@ def test_load_raw(): assert_image_similar_tofile(im, "Tests/images/drawing_wmf_ref.png", 2.0) +def test_load(): + with Image.open("Tests/images/drawing.emf") as im: + if hasattr(Image.core, "drawwmf"): + assert im.load()[0, 0] == (255, 255, 255) + + def test_register_handler(tmp_path): class TestHandler: methodCalled = False diff --git a/src/PIL/EpsImagePlugin.py b/src/PIL/EpsImagePlugin.py index 5d9920280..b67363beb 100644 --- a/src/PIL/EpsImagePlugin.py +++ b/src/PIL/EpsImagePlugin.py @@ -329,12 +329,12 @@ class EpsImageFile(ImageFile.ImageFile): def load(self, scale=1, transparency=False): # Load EPS via Ghostscript - if not self.tile: - return - self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) - self.mode = self.im.mode - self._size = self.im.size - self.tile = [] + if self.tile: + self.im = Ghostscript(self.tile, self.size, self.fp, scale, transparency) + self.mode = self.im.mode + self._size = self.im.size + self.tile = [] + return Image.Image.load(self) def load_seek(self, *args, **kwargs): # we can't incrementally load, so force ImageFile.parser to diff --git a/src/PIL/GbrImagePlugin.py b/src/PIL/GbrImagePlugin.py index 0f230602d..3d8fc47b2 100644 --- a/src/PIL/GbrImagePlugin.py +++ b/src/PIL/GbrImagePlugin.py @@ -84,12 +84,10 @@ class GbrImageFile(ImageFile.ImageFile): self._data_size = width * height * color_depth def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self._data_size)) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self._data_size)) + return Image.Image.load(self) # diff --git a/src/PIL/IcnsImagePlugin.py b/src/PIL/IcnsImagePlugin.py index 6412d1cfb..069aff96b 100644 --- a/src/PIL/IcnsImagePlugin.py +++ b/src/PIL/IcnsImagePlugin.py @@ -286,21 +286,22 @@ class IcnsImageFile(ImageFile.ImageFile): self.best_size[1] * self.best_size[2], ) - Image.Image.load(self) + px = Image.Image.load(self) if self.im and self.im.size == self.size: # Already loaded - return + return px self.load_prepare() # This is likely NOT the best way to do it, but whatever. im = self.icns.getimage(self.best_size) # If this is a PNG or JPEG 2000, it won't be loaded yet - im.load() + px = im.load() self.im = im.im self.mode = im.mode self.size = im.size - self.load_end() + + return px def _save(im, fp, filename): diff --git a/src/PIL/IcoImagePlugin.py b/src/PIL/IcoImagePlugin.py index d9ff9b5e7..82837f307 100644 --- a/src/PIL/IcoImagePlugin.py +++ b/src/PIL/IcoImagePlugin.py @@ -306,7 +306,7 @@ class IcoImageFile(ImageFile.ImageFile): def load(self): if self.im and self.im.size == self.size: # Already loaded - return + return Image.Image.load(self) im = self.ico.getimage(self.size) # if tile is PNG, it won't really be loaded yet im.load() diff --git a/src/PIL/ImageFile.py b/src/PIL/ImageFile.py index 3374a5b1d..331410f0e 100644 --- a/src/PIL/ImageFile.py +++ b/src/PIL/ImageFile.py @@ -328,6 +328,7 @@ class StubImageFile(ImageFile): # become the other object (!) self.__class__ = image.__class__ self.__dict__ = image.__dict__ + return image.load() def _load(self): """(Hook) Find actual image loader.""" diff --git a/src/PIL/WalImageFile.py b/src/PIL/WalImageFile.py index 1354ad32b..0dc695a88 100644 --- a/src/PIL/WalImageFile.py +++ b/src/PIL/WalImageFile.py @@ -51,14 +51,11 @@ class WalImageFile(ImageFile.ImageFile): self.info["next_name"] = next_name def load(self): - if self.im: - # Already loaded - return - - self.im = Image.core.new(self.mode, self.size) - self.frombytes(self.fp.read(self.size[0] * self.size[1])) - self.putpalette(quake2palette) - Image.Image.load(self) + if not self.im: + self.im = Image.core.new(self.mode, self.size) + self.frombytes(self.fp.read(self.size[0] * self.size[1])) + self.putpalette(quake2palette) + return Image.Image.load(self) def open(filename): diff --git a/src/PIL/WmfImagePlugin.py b/src/PIL/WmfImagePlugin.py index 27f5d2f87..c32cc52f8 100644 --- a/src/PIL/WmfImagePlugin.py +++ b/src/PIL/WmfImagePlugin.py @@ -158,7 +158,7 @@ class WmfStubImageFile(ImageFile.StubImageFile): (x1 - x0) * self.info["dpi"] // self._inch, (y1 - y0) * self.info["dpi"] // self._inch, ) - super().load() + return super().load() def _save(im, fp, filename): From 8e3878dec2b300f5b495dd81989a77089747abee Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 2 Feb 2022 14:33:19 +1100 Subject: [PATCH 37/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index d6cd9d50f..66d417393 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,12 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Enable arm64 for MSVC on Windows #5811 + [gaborkertesz-linaro, gaborkertesz] + +- Keep IPython/Jupyter text/plain output stable #5891 + [shamrin, radarhere] + - Raise an error when performing a negative crop #5972 [radarhere, hugovk] From dd46100bdc7fbb6c2fb71008a49c40f081eb0c7c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 10 Jan 2022 21:49:55 +1100 Subject: [PATCH 38/54] Restrict builtins within lambdas for ImageMath.eval --- Tests/test_imagemath.py | 12 ++++++++++-- src/PIL/ImageMath.py | 15 +++++++++++---- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Tests/test_imagemath.py b/Tests/test_imagemath.py index 25811aa89..39d91eade 100644 --- a/Tests/test_imagemath.py +++ b/Tests/test_imagemath.py @@ -52,9 +52,17 @@ def test_ops(): assert pixel(ImageMath.eval("float(B)**33", images)) == "F 8589934592.0" -def test_prevent_exec(): +@pytest.mark.parametrize( + "expression", + ( + "exec('pass')", + "(lambda: exec('pass'))()", + "(lambda: (lambda: exec('pass'))())()", + ), +) +def test_prevent_exec(expression): with pytest.raises(ValueError): - ImageMath.eval("exec('pass')") + ImageMath.eval(expression) def test_logical(): diff --git a/src/PIL/ImageMath.py b/src/PIL/ImageMath.py index 4b6e4ccda..09d9898d7 100644 --- a/src/PIL/ImageMath.py +++ b/src/PIL/ImageMath.py @@ -240,11 +240,18 @@ def eval(expression, _dict={}, **kw): if hasattr(v, "im"): args[k] = _Operand(v) - code = compile(expression, "", "eval") - for name in code.co_names: - if name not in args and name != "abs": - raise ValueError(f"'{name}' not allowed") + compiled_code = compile(expression, "", "eval") + def scan(code): + for const in code.co_consts: + if type(const) == type(compiled_code): + scan(const) + + for name in code.co_names: + if name not in args and name != "abs": + raise ValueError(f"'{name}' not allowed") + + scan(compiled_code) out = builtins.eval(expression, {"__builtins": {"abs": abs}}, args) try: return out.im From 8da80130dbc747f3954b4904247d26289fe722f9 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 17 Jan 2022 08:59:17 +1100 Subject: [PATCH 39/54] In show_file, use os.remove to remove temporary images --- Tests/test_imageshow.py | 6 +- src/PIL/ImageShow.py | 145 +++++++++++++++++++++++++++++++--------- 2 files changed, 119 insertions(+), 32 deletions(-) diff --git a/Tests/test_imageshow.py b/Tests/test_imageshow.py index 02edfdfa1..bf19a6033 100644 --- a/Tests/test_imageshow.py +++ b/Tests/test_imageshow.py @@ -85,11 +85,13 @@ def test_ipythonviewer(): not on_ci() or is_win32(), reason="Only run on CIs; hangs on Windows CIs", ) -def test_file_deprecated(): +def test_file_deprecated(tmp_path): + f = str(tmp_path / "temp.jpg") for viewer in ImageShow._viewers: + hopper().save(f) with pytest.warns(DeprecationWarning): try: - viewer.show_file(file="test.jpg") + viewer.show_file(file=f) except NotImplementedError: pass with pytest.raises(TypeError): diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 2165da307..7212baa1c 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -15,7 +15,6 @@ import os import shutil import subprocess import sys -import tempfile import warnings from shlex import quote @@ -180,16 +179,15 @@ class MacViewer(Viewer): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - subprocess.Popen( - ["im=$(cat); open -a Preview.app $im; sleep 20; rm -f $im"], - shell=True, - stdin=f, - ) - os.remove(temp_path) + subprocess.call(["open", "-a", "Preview.app", path]) + subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys, time;time.sleep(20);os.remove(sys.argv[1])", + path, + ] + ) return 1 @@ -205,6 +203,16 @@ class UnixViewer(Viewer): command = self.get_command_ex(file, **options)[0] return f"({command} {quote(file)}; rm -f {quote(file)})&" + +class XDGViewer(UnixViewer): + """ + The freedesktop.org ``xdg-open`` command. + """ + + def get_command_ex(self, file, **options): + command = executable = "xdg-open" + return command, executable + def show_file(self, path=None, **options): """ Display given file. @@ -223,28 +231,11 @@ class UnixViewer(Viewer): path = options.pop("file") else: raise TypeError("Missing required argument: 'path'") - fd, temp_path = tempfile.mkstemp() - with os.fdopen(fd, "w") as f: - f.write(path) - with open(temp_path) as f: - command = self.get_command_ex(path, **options)[0] - subprocess.Popen( - ["im=$(cat);" + command + " $im; rm -f $im"], shell=True, stdin=f - ) - os.remove(temp_path) + subprocess.Popen(["xdg-open", path]) + os.remove(path) return 1 -class XDGViewer(UnixViewer): - """ - The freedesktop.org ``xdg-open`` command. - """ - - def get_command_ex(self, file, **options): - command = executable = "xdg-open" - return command, executable - - class DisplayViewer(UnixViewer): """ The ImageMagick ``display`` command. @@ -257,6 +248,32 @@ class DisplayViewer(UnixViewer): command += f" -name {quote(title)}" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + args = ["display"] + if "title" in options: + args += ["-name", options["title"]] + args.append(path) + + subprocess.Popen(args) + os.remove(path) + return 1 + class GmDisplayViewer(UnixViewer): """The GraphicsMagick ``gm display`` command.""" @@ -266,6 +283,27 @@ class GmDisplayViewer(UnixViewer): command = "gm display" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + subprocess.Popen(["gm", "display", path]) + os.remove(path) + return 1 + class EogViewer(UnixViewer): """The GNOME Image Viewer ``eog`` command.""" @@ -275,6 +313,27 @@ class EogViewer(UnixViewer): command = "eog -n" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + subprocess.Popen(["eog", "-n", path]) + os.remove(path) + return 1 + class XVViewer(UnixViewer): """ @@ -290,6 +349,32 @@ class XVViewer(UnixViewer): command += f" -name {quote(title)}" return command, executable + def show_file(self, path=None, **options): + """ + Display given file. + + Before Pillow 9.1.0, the first argument was ``file``. This is now deprecated, + and ``path`` should be used instead. + """ + if path is None: + if "file" in options: + warnings.warn( + "The 'file' argument is deprecated and will be removed in Pillow " + "10 (2023-07-01). Use 'path' instead.", + DeprecationWarning, + ) + path = options.pop("file") + else: + raise TypeError("Missing required argument: 'path'") + args = ["xv"] + if "title" in options: + args += ["-name", options["title"]] + args.append(path) + + subprocess.Popen(args) + os.remove(path) + return 1 + if sys.platform not in ("win32", "darwin"): # unixoids if shutil.which("xdg-open"): From 143032103c9f2d55a0a7960bd3e630cb72549e8a Mon Sep 17 00:00:00 2001 From: Andrew Murray <3112309+radarhere@users.noreply.github.com> Date: Tue, 18 Jan 2022 11:24:01 +1100 Subject: [PATCH 40/54] Updated formatting Co-authored-by: Hugo van Kemenade --- src/PIL/ImageShow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index 7212baa1c..ccdb0b2a0 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -184,7 +184,7 @@ class MacViewer(Viewer): [ sys.executable, "-c", - "import os, sys, time;time.sleep(20);os.remove(sys.argv[1])", + "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", path, ] ) From 10c4f75aaa383bd9671e923e3b91d391ea12d781 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 08:58:12 +1100 Subject: [PATCH 41/54] Added delay after opening image with xdg-open --- src/PIL/ImageShow.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/PIL/ImageShow.py b/src/PIL/ImageShow.py index ccdb0b2a0..f8829fc21 100644 --- a/src/PIL/ImageShow.py +++ b/src/PIL/ImageShow.py @@ -126,6 +126,16 @@ class Viewer: os.system(self.get_command(path, **options)) return 1 + def _remove_path_after_delay(self, path): + subprocess.Popen( + [ + sys.executable, + "-c", + "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", + path, + ] + ) + # -------------------------------------------------------------------- @@ -180,14 +190,7 @@ class MacViewer(Viewer): else: raise TypeError("Missing required argument: 'path'") subprocess.call(["open", "-a", "Preview.app", path]) - subprocess.Popen( - [ - sys.executable, - "-c", - "import os, sys, time; time.sleep(20); os.remove(sys.argv[1])", - path, - ] - ) + self._remove_path_after_delay(path) return 1 @@ -232,7 +235,7 @@ class XDGViewer(UnixViewer): else: raise TypeError("Missing required argument: 'path'") subprocess.Popen(["xdg-open", path]) - os.remove(path) + self._remove_path_after_delay(path) return 1 From 596eaf35cc983fe73e2408e3ca7f3e1431fd06e3 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 09:46:57 +1100 Subject: [PATCH 42/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index 66d417393..fc9455652 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -32,6 +32,15 @@ Changelog (Pillow) - Remove readonly from Image.__eq__ #5930 [hugovk] +9.0.1 (2022-02-03) +------------------ + +- In show_file, use os.remove to remove temporary images. CVE-2022-24303 #6010 + [radarhere, hugovk] + +- Restrict builtins within lambdas for ImageMath.eval. CVE-2022-22817 #6009 + [radarhere] + 9.0.0 (2022-01-02) ------------------ From 8ef2d987ab652346126cf7e567eac7d2c0754940 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 09:48:56 +1100 Subject: [PATCH 43/54] Added release notes for 9.0.1 --- docs/releasenotes/9.0.1.rst | 23 +++++++++++++++++++++++ docs/releasenotes/index.rst | 1 + 2 files changed, 24 insertions(+) create mode 100644 docs/releasenotes/9.0.1.rst diff --git a/docs/releasenotes/9.0.1.rst b/docs/releasenotes/9.0.1.rst new file mode 100644 index 000000000..5d1b246bc --- /dev/null +++ b/docs/releasenotes/9.0.1.rst @@ -0,0 +1,23 @@ +9.0.1 +----- + +Security +======== + +This release addresses several security problems. + +:cve:`CVE-2022-24303`: If the path to the temporary directory on Linux or macOS +contained a space, this would break removal of the temporary image file after +``im.show()`` (and related actions), and potentially remove an unrelated file. This +been present since PIL. + +:cve:`CVE-2022-22817`: While Pillow 9.0 restricted top-level builtins available to +:py:meth:`PIL.ImageMath.eval`, it did not prevent builtins available to lambda +expressions. These are now also restricted. + +Other Changes +============= + +Pillow 9.0 added support for ``xdg-open`` as an image viewer, but there have been +reports that the temporary image file was removed too quickly to be loaded into the +final application. A delay has been added. diff --git a/docs/releasenotes/index.rst b/docs/releasenotes/index.rst index 8d1ad7837..e9b11c220 100644 --- a/docs/releasenotes/index.rst +++ b/docs/releasenotes/index.rst @@ -14,6 +14,7 @@ expected to be backported to earlier versions. .. toctree:: :maxdepth: 2 + 9.0.1 9.0.0 8.4.0 8.3.2 From 9d8f173e39458bb73107f90813c745717d87ea78 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 3 Feb 2022 18:05:50 +1100 Subject: [PATCH 44/54] Updated libimagequant to 4.0.0 --- depends/install_imagequant.sh | 11 ++++++----- docs/installation.rst | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/depends/install_imagequant.sh b/depends/install_imagequant.sh index 774f26767..31fc2adaa 100755 --- a/depends/install_imagequant.sh +++ b/depends/install_imagequant.sh @@ -1,14 +1,15 @@ #!/bin/bash # install libimagequant -archive=libimagequant-2.17.0 +archive=libimagequant-4.0.0 ./download-and-extract.sh $archive https://raw.githubusercontent.com/python-pillow/pillow-depends/main/$archive.tar.gz -pushd $archive +pushd $archive/imagequant-sys -make shared -sudo cp libimagequant.so* /usr/lib/ -sudo cp libimagequant.h /usr/include/ +cargo install cargo-c +cargo cinstall --prefix=/usr --destdir=. +sudo cp usr/lib/libimagequant.so* /usr/lib/ +sudo cp usr/include/libimagequant.h /usr/include/ popd diff --git a/docs/installation.rst b/docs/installation.rst index 984a689c2..a6ed8f681 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -187,7 +187,7 @@ Many of Pillow's features require external libraries: * **libimagequant** provides improved color quantization - * Pillow has been tested with libimagequant **2.6-2.17.0** + * Pillow has been tested with libimagequant **2.6-4.0** * Libimagequant is licensed GPLv3, which is more restrictive than the Pillow license, therefore we will not be distributing binaries with libimagequant support enabled. From caf6fc60cac8ed59b8b2c361df1e9705be3d0504 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 4 Feb 2022 03:19:49 +1100 Subject: [PATCH 45/54] Corrected sentence [ci skip] --- docs/releasenotes/9.0.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releasenotes/9.0.1.rst b/docs/releasenotes/9.0.1.rst index 5d1b246bc..c1feee088 100644 --- a/docs/releasenotes/9.0.1.rst +++ b/docs/releasenotes/9.0.1.rst @@ -9,7 +9,7 @@ This release addresses several security problems. :cve:`CVE-2022-24303`: If the path to the temporary directory on Linux or macOS contained a space, this would break removal of the temporary image file after ``im.show()`` (and related actions), and potentially remove an unrelated file. This -been present since PIL. +has been present since PIL. :cve:`CVE-2022-22817`: While Pillow 9.0 restricted top-level builtins available to :py:meth:`PIL.ImageMath.eval`, it did not prevent builtins available to lambda From 943479941578b78ed3f8ea6bd1e3031b1244f006 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Fri, 4 Feb 2022 03:29:26 +1100 Subject: [PATCH 46/54] Updated lcms2 to 2.13.1 --- docs/installation.rst | 2 +- winbuild/build_prepare.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index a6ed8f681..023d27ddd 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -169,7 +169,7 @@ Many of Pillow's features require external libraries: * **littlecms** provides color management * Pillow version 2.2.1 and below uses liblcms1, Pillow 2.3.0 and - above uses liblcms2. Tested with **1.19** and **2.7-2.13**. + above uses liblcms2. Tested with **1.19** and **2.7-2.13.1**. * **libwebp** provides the WebP format. diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 01f1bac29..6a37ff8bb 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -221,9 +221,9 @@ deps = { # "bins": [r"objs\{msbuild_arch}\Release\freetype.dll"], }, "lcms2": { - "url": SF_MIRROR + "/project/lcms/lcms/2.13/lcms2-2.13.tar.gz", - "filename": "lcms2-2.13.tar.gz", - "dir": "lcms2-2.13", + "url": SF_MIRROR + "/project/lcms/lcms/2.13/lcms2-2.13.1.tar.gz", + "filename": "lcms2-2.13.1.tar.gz", + "dir": "lcms2-2.13.1", "patch": { r"Projects\VC2019\lcms2_static\lcms2_static.vcxproj": { # default is /MD for x86 and /MT for x64, we need /MD always From f3762905e08a96902f20391289d487b028e68f98 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Sun, 6 Feb 2022 08:32:37 +1100 Subject: [PATCH 47/54] Upgraded Python 3.10 image to VS 2022 --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index d525e4cfc..f86500b48 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -12,7 +12,7 @@ environment: matrix: - PYTHON: C:/Python310 ARCHITECTURE: x86 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2022 - PYTHON: C:/Python37-x64 ARCHITECTURE: x64 APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 From 718b72c0fd949ba6cf11bc5ffd936e772132231a Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 7 Feb 2022 08:13:20 +1100 Subject: [PATCH 48/54] Updated harfbuzz to 3.3.2 --- winbuild/build_prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/winbuild/build_prepare.py b/winbuild/build_prepare.py index 01f1bac29..b00905b09 100644 --- a/winbuild/build_prepare.py +++ b/winbuild/build_prepare.py @@ -280,9 +280,9 @@ deps = { "libs": [r"imagequant.lib"], }, "harfbuzz": { - "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.1.zip", - "filename": "harfbuzz-3.3.1.zip", - "dir": "harfbuzz-3.3.1", + "url": "https://github.com/harfbuzz/harfbuzz/archive/3.3.2.zip", + "filename": "harfbuzz-3.3.2.zip", + "dir": "harfbuzz-3.3.2", "build": [ cmd_cmake("-DHB_HAVE_FREETYPE:BOOL=TRUE"), cmd_nmake(target="clean"), From 85b872deb650847f9b087db53b6d4bb963679653 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Mon, 7 Feb 2022 10:18:14 +1100 Subject: [PATCH 49/54] Added unpacker from RGBA;15 to RGB --- src/libImaging/Unpack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libImaging/Unpack.c b/src/libImaging/Unpack.c index 5dac95c1d..4f9838fa8 100644 --- a/src/libImaging/Unpack.c +++ b/src/libImaging/Unpack.c @@ -1529,6 +1529,7 @@ static struct { {"RGB", "RGBX", 32, copy4}, {"RGB", "RGBX;L", 32, unpackRGBAL}, {"RGB", "RGBA;L", 32, unpackRGBAL}, + {"RGB", "RGBA;15", 16, ImagingUnpackRGBA15}, {"RGB", "BGRX", 32, ImagingUnpackBGRX}, {"RGB", "XRGB", 32, ImagingUnpackXRGB}, {"RGB", "XBGR", 32, ImagingUnpackXBGR}, From a278e0aa656745122e76c21d1589e9c9762ca743 Mon Sep 17 00:00:00 2001 From: nulano Date: Mon, 7 Feb 2022 23:57:35 +0000 Subject: [PATCH 50/54] issue warning if Raqm layout is requested, but Raqm is not available --- Tests/test_imagefont.py | 13 +++++++++++++ src/PIL/ImageFont.py | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/Tests/test_imagefont.py b/Tests/test_imagefont.py index 0d423aab7..3dcbf18d2 100644 --- a/Tests/test_imagefont.py +++ b/Tests/test_imagefont.py @@ -1022,3 +1022,16 @@ def test_oom(test_file): font = ImageFont.truetype(BytesIO(f.read())) with pytest.raises(Image.DecompressionBombError): font.getmask("Test Text") + + +def test_raqm_missing_warning(monkeypatch): + monkeypatch.setattr(ImageFont.core, "HAVE_RAQM", False) + with pytest.warns(UserWarning) as record: + font = ImageFont.truetype( + FONT_PATH, FONT_SIZE, layout_engine=ImageFont.LAYOUT_RAQM + ) + assert font.layout_engine == ImageFont.LAYOUT_BASIC + assert str(record[-1].message) == ( + "Raqm layout was requested, but Raqm is not available. " + "Falling back to basic layout." + ) diff --git a/src/PIL/ImageFont.py b/src/PIL/ImageFont.py index 805c8fff9..58bf46a32 100644 --- a/src/PIL/ImageFont.py +++ b/src/PIL/ImageFont.py @@ -169,6 +169,12 @@ class FreeTypeFont: if core.HAVE_RAQM: layout_engine = LAYOUT_RAQM elif layout_engine == LAYOUT_RAQM and not core.HAVE_RAQM: + import warnings + + warnings.warn( + "Raqm layout was requested, but Raqm is not available. " + "Falling back to basic layout." + ) layout_engine = LAYOUT_BASIC self.layout_engine = layout_engine From 92371504316194883518dd5f0182ea01cbbab0f7 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Tue, 8 Feb 2022 23:35:01 +1100 Subject: [PATCH 51/54] Added CentOS Stream 9 --- .github/workflows/test-docker.yml | 1 + docs/installation.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/test-docker.yml b/.github/workflows/test-docker.yml index 656df5e91..db866774c 100644 --- a/.github/workflows/test-docker.yml +++ b/.github/workflows/test-docker.yml @@ -20,6 +20,7 @@ jobs: arch, centos-7-amd64, centos-stream-8-amd64, + centos-stream-9-amd64, debian-10-buster-x86, debian-11-bullseye-x86, fedora-34-amd64, diff --git a/docs/installation.rst b/docs/installation.rst index 023d27ddd..cf94f0c72 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -455,6 +455,8 @@ These platforms are built and tested for every change. +----------------------------------+----------------------------+---------------------+ | CentOS Stream 8 | 3.9 | x86-64 | +----------------------------------+----------------------------+---------------------+ +| CentOS Stream 9 | 3.9 | x86-64 | ++----------------------------------+----------------------------+---------------------+ | Debian 10 Buster | 3.7 | x86 | +----------------------------------+----------------------------+---------------------+ | Debian 11 Bullseye | 3.9 | x86 | From d7922d1e85186951dbd1e1ac856a0cb45c4a7e06 Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Wed, 9 Feb 2022 14:27:21 +1100 Subject: [PATCH 52/54] Updated macOS tested Pillow versions [ci skip] --- docs/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index cf94f0c72..141649107 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -496,11 +496,11 @@ These platforms have been reported to work at the versions mentioned. | Operating system | | Tested Python | | Latest tested | | Tested | | | | versions | | Pillow version | | processors | +==================================+===========================+==================+==============+ -| macOS 12 Big Sur | 3.7, 3.8, 3.9, 3.10 | 9.0.0 |arm | +| macOS 12 Big Sur | 3.7, 3.8, 3.9, 3.10 | 9.0.1 |arm | +----------------------------------+---------------------------+------------------+--------------+ | macOS 11 Big Sur | 3.7, 3.8, 3.9, 3.10 | 8.4.0 |arm | | +---------------------------+------------------+--------------+ -| | 3.7, 3.8, 3.9, 3.10 | 9.0.0 |x86-64 | +| | 3.7, 3.8, 3.9, 3.10 | 9.0.1 |x86-64 | | +---------------------------+------------------+--------------+ | | 3.6 | 8.4.0 |x86-64 | +----------------------------------+---------------------------+------------------+--------------+ From 601c9d8515dba996af3f0b96d1a671619de37f10 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 9 Feb 2022 14:28:43 +0200 Subject: [PATCH 53/54] Fix return in docs --- src/PIL/JpegImagePlugin.py | 1 + src/PIL/PngImagePlugin.py | 1 + src/PIL/TiffImagePlugin.py | 1 + 3 files changed, 3 insertions(+) diff --git a/src/PIL/JpegImagePlugin.py b/src/PIL/JpegImagePlugin.py index ccdcc20a8..b9999bdaf 100644 --- a/src/PIL/JpegImagePlugin.py +++ b/src/PIL/JpegImagePlugin.py @@ -482,6 +482,7 @@ class JpegImageFile(ImageFile.ImageFile): """ Returns a dictionary containing the XMP tags. Requires defusedxml to be installed. + :returns: XMP tags in a dictionary. """ diff --git a/src/PIL/PngImagePlugin.py b/src/PIL/PngImagePlugin.py index 0f596f1fd..ae38b0ed3 100644 --- a/src/PIL/PngImagePlugin.py +++ b/src/PIL/PngImagePlugin.py @@ -982,6 +982,7 @@ class PngImageFile(ImageFile.ImageFile): """ Returns a dictionary containing the XMP tags. Requires defusedxml to be installed. + :returns: XMP tags in a dictionary. """ return ( diff --git a/src/PIL/TiffImagePlugin.py b/src/PIL/TiffImagePlugin.py index e54082fec..a3922d699 100644 --- a/src/PIL/TiffImagePlugin.py +++ b/src/PIL/TiffImagePlugin.py @@ -1124,6 +1124,7 @@ class TiffImageFile(ImageFile.ImageFile): """ Returns a dictionary containing the XMP tags. Requires defusedxml to be installed. + :returns: XMP tags in a dictionary. """ return self._getxmp(self.tag_v2[700]) if 700 in self.tag_v2 else {} From 4e65e2c29de489ca098399c5c179168bd73ef17c Mon Sep 17 00:00:00 2001 From: Andrew Murray Date: Thu, 10 Feb 2022 09:47:43 +1100 Subject: [PATCH 54/54] Update CHANGES.rst [ci skip] --- CHANGES.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index fc9455652..900346f74 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,6 +5,9 @@ Changelog (Pillow) 9.1.0 (unreleased) ------------------ +- Added unpacker from RGBA;15 to RGB #6031 + [radarhere] + - Enable arm64 for MSVC on Windows #5811 [gaborkertesz-linaro, gaborkertesz]