From 4214abbdc827fc2e39010766607afa1eb6adbcca Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 5 Feb 2024 09:11:28 +0100 Subject: [PATCH 1/6] Fix ffprobe test failing on Alpine --- tests/video/test_video.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/video/test_video.py b/tests/video/test_video.py index 5b151f2e..ef541112 100644 --- a/tests/video/test_video.py +++ b/tests/video/test_video.py @@ -130,7 +130,17 @@ def test_get_media_info(media_format, media, expected, test_files): with tempfile.TemporaryDirectory() as t: src = pathlib.Path(t).joinpath(media) shutil.copy2(test_files[media_format], src) - assert get_media_info(src) == expected + result = get_media_info(src) + assert result.keys() == expected.keys() + assert result["codecs"] == expected["codecs"] + assert result["duration"] == expected["duration"] + # for bitrate, we need to allow some variability, not all ffmpeg version are + # reporting the same values (e.g. Alpine Linux is reporting 3837275 instead of + # 3818365 for video.mp4) ; we allow 1% variability with following assertion + assert ( + abs(100.0 * (result["bitrate"] - expected["bitrate"]) / expected["bitrate"]) + < 1 + ) def test_preset_has_mime_and_ext(): From cb2bad14dfd55304cbfb44e6559691dc93103137 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 11:42:10 +0100 Subject: [PATCH 2/6] Move to python-magic for alpine compatibility python-magic is supporting Alpine Linux since 0.4.24 (hence the minimum version) while file-magic is still not supporting it. --- pyproject.toml | 2 +- src/zimscraperlib/filesystem.py | 6 +++++- tests/filesystem/test_filesystem.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 24d64bf0..2fec6ce0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "colorthief==0.2.1", "python-resize-image>=1.1.19,<1.2", "Babel>=2.9,<3.0", - "file-magic>=0.4.0,<0.5", + "python-magic>=0.4.24,<0.5", "libzim>=3.4.0,<4.0", "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done diff --git a/src/zimscraperlib/filesystem.py b/src/zimscraperlib/filesystem.py index 509675eb..7b22dfe3 100644 --- a/src/zimscraperlib/filesystem.py +++ b/src/zimscraperlib/filesystem.py @@ -31,7 +31,11 @@ def get_content_mimetype(content: bytes) -> str: """MIME Type of content retrieved from magic headers""" try: - detected_mime = magic.detect_from_content(content).mime_type + detected_mime = magic.from_buffer(content, mime=True) + if isinstance( + detected_mime, bytes + ): # pragma: no cover (old python-magic versions where returning bytes) + detected_mime = detected_mime.decode() except UnicodeDecodeError: return "application/octet-stream" return MIME_OVERRIDES.get(detected_mime, detected_mime) diff --git a/tests/filesystem/test_filesystem.py b/tests/filesystem/test_filesystem.py index 71daba63..08579c16 100644 --- a/tests/filesystem/test_filesystem.py +++ b/tests/filesystem/test_filesystem.py @@ -28,10 +28,10 @@ def test_content_mimetype_fallback(monkeypatch, undecodable_byte_stream): assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" # mock then so we keep coverage on systems where magic works - def raising_magic(*args): # noqa: ARG001 + def raising_magic(*args, **kwargs): # noqa: ARG001 raise UnicodeDecodeError("nocodec", b"", 0, 1, "noreason") - monkeypatch.setattr(magic, "detect_from_content", raising_magic) + monkeypatch.setattr(magic, "from_buffer", raising_magic) assert get_content_mimetype(undecodable_byte_stream) == "application/octet-stream" From 28647a58faf351cb0ece361820bfb72554804b23 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Mon, 12 Feb 2024 14:27:13 +0100 Subject: [PATCH 3/6] Instructions for Alpine Linux --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 1d8870e7..dcd71ae4 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ sudo apt install libmagic1 wget ffmpeg \ libharfbuzz-dev libfribidi-dev libxcb1-dev gifsicle ``` +## Alpine +``` +apk add ffmpeg gifsicle libmagic wget libjpeg +``` + # Contribution This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing) From f3251af53d632f5327a9cb18de4ea8a7fe196736 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 08:45:58 +0100 Subject: [PATCH 4/6] Update CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2fb1a8b..fa3bbf95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Using openZIM Python bootstrap conventions (including hatch-openzim plugin) #120 -- Suuport for Python 3.12, drop Python 3.7 #118 -- Replace "iso-369" iso639-lang by "iso639-lang" library +- Add support for Python 3.12, drop Python 3.7 support #118 +- Replace "iso-369" by "iso639-lang" library +- Replace "file-magic" by "python-magic" library for Alpine Linux support and better maintenance - Rework the VideoWebmLow preset for faster encoding and smaller file size (preset has been bumped to version 2) - When reencoding a video, ffmpeg now uses only 1 CPU thread by default (new arg to `reencode` allows to override this default value) From d229055fc3f5a13495153da1c073df8dfbfe9aa8 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 15:52:35 +0100 Subject: [PATCH 5/6] Minimal supported python-magic version is 0.4.3 indeed --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2fec6ce0..53ad3b1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "colorthief==0.2.1", "python-resize-image>=1.1.19,<1.2", "Babel>=2.9,<3.0", - "python-magic>=0.4.24,<0.5", + "python-magic>=0.4.3,<0.5", "libzim>=3.4.0,<4.0", "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done From 68d3e30806645a30272ce54c79356925f1fe4e5a Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 13 Feb 2024 15:55:47 +0100 Subject: [PATCH 6/6] Explicitely mention i18n not working on Alpine --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dcd71ae4..e909352f 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ sudo apt install libmagic1 wget ffmpeg \ apk add ffmpeg gifsicle libmagic wget libjpeg ``` +**Nota:** i18n features do not work on Alpine, see https://github.com/openzim/python-scraperlib/issues/134 ; there is one corresponding test which is failing. + # Contribution This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing)