Memory allocation error (with huge CPU usage) with a picture too big

[gu1lhem]

How to use GitHub

• Please use the 👍 reaction to show that you are affected by the same issue.
• Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
• Subscribe to receive notifications on status change and new comments.

---

Steps to reproduce

1. Upload a 200Mpx image made with a Samsung S24 Ultra or S25 Ultra
2. Try to generate a preview

Expected behaviour

The preview should work

Actual behaviour

Using the OCC command, the response is:

In Generator.php line 366:

  [OCP\Files\NotFoundException]
  No provider successfully handled the preview generation


Exception trace:
  at /var/www/html/lib/private/Preview/Generator.php:366
 OC\Preview\Generator->generateProviderPreview() at /var/www/html/lib/private/Preview/Generator.php:313
 OC\Preview\Generator->getMaxPreview() at /var/www/html/lib/private/Preview/Generator.php:117
 OC\Preview\Generator->generatePreviews() at /var/www/html/lib/private/PreviewManager.php:182
 OC\PreviewManager->generatePreviews() at /var/www/html/core/Command/Preview/Generate.php:92
 OC\Core\Command\Preview\Generate->execute() at /var/www/html/3rdparty/symfony/console/Command/Command.php:326
 Symfony\Component\Console\Command\Command->run() at /var/www/html/3rdparty/symfony/console/Application.php:1078
 Symfony\Component\Console\Application->doRunCommand() at /var/www/html/3rdparty/symfony/console/Application.php:324
 Symfony\Component\Console\Application->doRun() at /var/www/html/3rdparty/symfony/console/Application.php:175
 Symfony\Component\Console\Application->run() at /var/www/html/lib/private/Console/Application.php:187
 OC\Console\Application->run() at /var/www/html/console.php:87
 require_once() at /var/www/html/occ:33

preview:generate [-s|--size SIZE] [-c|--crop] [-m|--mode MODE] [--] <file>

In Nextcloud logs, I get Memory allocation error: Security limit exceeded: Allocating (size) exceeds the security limit of 536870912 bytes (6.1000)

The CPU usage then stays huge.

Server configuration

Web server: Apache

Database: pgsql

PHP version: 8.3.22

Nextcloud version: Nextcloud AIO v11.1.0

List of activated apps

  - activity: 4.0.0
  - admin_audit: 1.21.0
  - bruteforcesettings: 4.0.0
  - calendar: 5.3.3
  - cfg_share_links: 7.0.1
  - circles: 31.0.0
  - cloud_federation_api: 1.14.0
  - cloud_py_api: 0.2.0
  - comments: 1.21.0
  - contacts: 7.1.3
  - contactsinteraction: 1.12.0
  - cookbook: 0.11.3
  - cospend: 3.0.11
  - dashboard: 7.11.0
  - dav: 1.33.0
  - deck: 1.15.1
  - duplicatefinder: 1.7.3
  - epubviewer: 1.8.0
  - federatedfilesharing: 1.21.0
  - federation: 1.21.0
  - files: 2.3.1
  - files_downloadlimit: 4.0.0
  - files_external: 1.23.0
  - files_fulltextsearch: 31.0.0
  - files_pdfviewer: 4.0.0
  - files_reminders: 1.4.0
  - files_sharing: 1.23.1
  - files_trashbin: 1.21.0
  - files_versions: 1.24.0
  - firstrunwizard: 4.0.0
  - fulltextsearch: 31.0.0
  - fulltextsearch_elasticsearch: 31.0.0
  - geoblocker: 0.5.16
  - gpxpod: 7.0.4
  - group_everyone: 0.1.17
  - logreader: 4.0.0
  - lookup_server_connector: 1.19.0
  - mail: 5.1.4
  - mediadc: 0.4.0
  - memories: 7.5.2
  - music: 2.2.0
  - nextcloud-aio: 0.8.0
  - nextcloud_announcements: 3.0.0
  - notes: 4.12.1
  - notifications: 4.0.0
  - notify_push: 1.1.0
  - oauth2: 1.19.1
  - password_policy: 3.0.0
  - photos: 4.0.0-dev.1
  - previewgenerator: 5.8.0
  - privacy: 3.0.0
  - profile: 1.0.0
  - provisioning_api: 1.21.0
  - recognize: 9.0.1
  - recommendations: 4.0.0
  - related_resources: 2.0.0
  - richdocuments: 8.7.1
  - secrets: 2.1.2
  - serverinfo: 3.0.0
  - settings: 1.14.0
  - sharebymail: 1.21.0
  - spreed: 21.1.0
  - support: 3.0.0
  - suspicious_login: 9.0.1
  - systemtags: 1.21.1
  - tasks: 0.16.1
  - text: 5.0.0
  - theming: 2.6.1
  - twofactor_backupcodes: 1.20.0
  - twofactor_totp: 13.0.0-dev.0
  - twofactor_webauthn: 2.2.0
  - updatenotification: 1.21.0
  - user_status: 1.11.0
  - viewer: 4.0.0
  - weather_status: 1.11.0
  - webhook_listeners: 1.2.0
  - whiteboard: 1.1.1
  - workflowengine: 2.13.0

Nextcloud configuration

{
    "system": {
        "one-click-instance": true,
        "one-click-instance.user-limit": 100,
        "memcache.local": "\\OC\\Memcache\\APCu",
        "apps_paths": [
            {
                "path": "\/var\/www\/html\/apps",
                "url": "\/apps",
                "writable": false
            },
            {
                "path": "\/var\/www\/html\/custom_apps",
                "url": "\/custom_apps",
                "writable": true
            }
        ],
        "appsallowlist": false,
        "check_data_directory_permissions": false,
        "memcache.distributed": "\\OC\\Memcache\\Redis",
        "memcache.locking": "\\OC\\Memcache\\Redis",
        "redis": {
            "host": "***REMOVED SENSITIVE VALUE***",
            "password": "***REMOVED SENSITIVE VALUE***",
            "port": 6379
        },
        "overwritehost": "***REMOVED SENSITIVE VALUE***",
        "overwriteprotocol": "https",
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [
            "***REMOVED SENSITIVE VALUE***"
        ],
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "pgsql",
        "version": "31.0.6.2",
        "overwrite.cli.url": "***REMOVED SENSITIVE VALUE***",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "",
        "dbtableprefix": "oc_",
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "installed": true,
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "maintenance": false,
        "updatedirectory": "\/nc-updater",
        "loglevel": 0,
        "log_type": "file",
        "logfile": "\/var\/www\/html\/data\/nextcloud.log",
        "log_rotate_size": "10485760",
        "log.condition": {
            "apps": [
                "admin_audit"
            ]
        },
        "preview_max_x": 2048,
        "preview_max_y": 2048,
        "preview_max_filesize_image": -1,
        "preview_max_memory": -1,
        "jpeg_quality": 60,
        "enabledPreviewProviders": {
            "0": "OC\\Preview\\Imaginary",
            "1": "OC\\Preview\\MarkDown",
            "2": "OC\\Preview\\MP3",
            "3": "OC\\Preview\\TXT",
            "4": "OC\\Preview\\OpenDocument",
            "5": "OC\\Preview\\Movie",
            "6": "OC\\Preview\\Krita",
            "7": "OC\\Preview\\ImaginaryPDF",
            "8": "OC\\Preview\\HEIC",
            "9": "OC\\Preview\\TIFF",
            "23": "OC\\Preview\\ImaginaryPDF"
        },
        "enable_previews": true,
        "upgrade.disable-web": true,
        "mail_smtpmode": "smtp",
        "trashbin_retention_obligation": "auto, 30",
        "versions_retention_obligation": "auto, 30",
        "activity_expire_days": "30",
        "simpleSignUpLink.shown": false,
        "share_folder": "\/Shared",
        "one-click-instance.link": "https:\/\/nextcloud.com\/all-in-one\/",
        "upgrade.cli-upgrade-link": "https:\/\/github.com\/nextcloud\/all-in-one\/discussions\/2726",
        "allow_local_remote_servers": true,
        "davstorage.request_timeout": 7200,
        "htaccess.RewriteBase": "\/",
        "dbpersistent": false,
        "files_external_allow_create_new_local": false,
        "trusted_proxies": "***REMOVED SENSITIVE VALUE***",
        "preview_imaginary_url": "***REMOVED SENSITIVE VALUE***",
        "defaultapp": "",
        "twofactor_enforced": "false",
        "twofactor_enforced_groups": [],
        "twofactor_enforced_excluded_groups": [],
        "maintenance_window_start": 100,
        "preview_imaginary_key": "***REMOVED SENSITIVE VALUE***",
        "auth.bruteforce.protection.enabled": true,
        "ratelimit.protection.enabled": true,
        "default_phone_region": "FR",
        "mail_smtpname": "***REMOVED SENSITIVE VALUE***",
        "mail_smtppassword": "***REMOVED SENSITIVE VALUE***",
        "mail_domain": "***REMOVED SENSITIVE VALUE***",
        "mail_from_address": "***REMOVED SENSITIVE VALUE***",
        "mail_smtphost": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpauth": 1,
        "mail_smtpport": "465",
        "mail_sendmailmode": "smtp",
        "mail_smtpsecure": "ssl",
        "documentation_url.server_logs": "https:\/\/github.com\/nextcloud\/all-in-one\/discussions\/5425",
        "memories.db.triggers.fcu": true,
        "memories.exiftool": "\/var\/www\/html\/custom_apps\/memories\/bin-ext\/exiftool-aarch64-musl",
        "memories.vod.path": "\/var\/www\/html\/custom_apps\/memories\/bin-ext\/go-vod-aarch64",
        "memories.vod.ffmpeg": "\/usr\/bin\/ffmpeg",
        "memories.vod.ffprobe": "\/usr\/bin\/ffprobe",
        "updatechecker": false,
        "DOMAIN": "***REMOVED SENSITIVE VALUE***"
    }
}

The issue seems to be coming from sharp and libvips according to my researchs. It has been discussed on immich:

• Images that fail to generate thumbnail do not appear in the web view while uploading immich-app/immich#17500
• Error: heif: Unsupported feature: Unsupported color conversion (4.3003) immich-app/immich#14316

Here are some MR on libvips that seem to talk about this: libvips/libvips#4398 libvips/libvips#4399
Is this related ?

[maxi1134]

I've also encountered this issue with 200MP pictures!

[st3iny]

This is likely a problem with your image library linking against libheif (e.g., Imagick) and setting an insufficient maximum memory limit, not Nextcloud itself. Ref ImageMagick/ImageMagick#8109

@gu1lhem @maxi1134 @CodyTray Could one of you please run the following command on their Nextcloud server/container:
magick --version

If your version is new enough, the problem should be fixed by simply adjusting the policy of Imagick (to allow bigger allocations for pictures). You might also need to increase the maximum allowed memory size for PHP accordingly.

Regarding libvips: I assume you are using the Imaginay container to generate previews? We might be out of luck since this requires updating the library inside the container as the referenced PRs are rather recent. However, the Imaginary project is pretty much dead at this point so you would need to do it yourself or hope for a fork.

[maxi1134]

@gu1lhem @maxi1134 @CodyTray Could one of you please run the following command on their Nextcloud server/container:
magick --version

maxi1134@diaspora:~$ sudo docker exec -it nextcloud-app-1  /bin/bash
[sudo] password for maxi1134:
root@b85b6a65474d:/var/www/html# magick --version
Version: ImageMagick 7.1.1-43 Q16 x86_64 22550 https://imagemagick.org
Copyright: (C) 1999 ImageMagick Studio LLC
License: https://imagemagick.org/script/license.php
Features: Cipher DPC Modules OpenMP(4.5)
Delegates (built-in): bzlib djvu fftw fontconfig freetype heic jbig jng jp2 jpeg lcms lqr ltdl lzma openexr pangocairo png raw tiff webp wmf x xml zlib zstd
Compiler: gcc (14.2)

Here is the result!

[maxi1134]

Also, this is my current policy:

<policymap>
  <policy domain="Undefined" rights="none"/>
  <!-- Set maximum parallel threads. -->
  <!-- <policy domain="resource" name="thread" value="2"/> -->
  <!-- Set maximum time in seconds. When this limit is exceeded, an exception
       is thrown and processing stops. -->
  <!-- <policy domain="resource" name="time" value="120"/> -->
  <!-- Set maximum number of open pixel cache files. When this limit is
       exceeded, any subsequent pixels cached to disk are closed and reopened
       on demand. -->
  <!-- <policy domain="resource" name="file" value="768"/> -->
  <policy domain="resource" name="memory" value="10240MiB"/>
  <!-- Set maximum amount of memory map in bytes to allocate for the pixel
       cache. When this limit is exceeded, the image pixels are cached to
       disk. -->
  <policy domain="resource" name="map" value="20480MiB"/>
  <!-- Set the maximum width * height of an image that can reside in the pixel
       cache memory. Images that exceed the area limit are cached to disk. -->
  <policy domain="resource" name="area" value="2560MP"/>
  <!-- Set maximum amount of disk space in bytes permitted for use by the pixel
       cache. When this limit is exceeded, the pixel cache is not be created
       and an exception is thrown. -->
  <policy domain="resource" name="disk" value="20GiB"/>
  <!-- Set the maximum length of an image sequence.  When this limit is
       exceeded, an exception is thrown. -->
  <!-- <policy domain="resource" name="list-length" value="32"/> -->
  <!-- Set the maximum width of an image.  When this limit is exceeded, an
       exception is thrown. -->
  <policy domain="resource" name="width" value="320KP"/>
  <!-- Set the maximum height of an image.  When this limit is exceeded, an
       exception is thrown. -->
  <policy domain="resource" name="height" value="320KP"/>
  <!-- Periodically yield the CPU for at least the time specified in
       milliseconds. -->
  <!-- <policy domain="resource" name="throttle" value="2"/> -->
  <!-- Do not create temporary files in the default shared directories, instead
      specify a private area to store only ImageMagick temporary files. -->
  <!-- <policy domain="resource" name="temporary-path" value="/magick/tmp/"/> -->
  <!-- Force memory initialization by memory mapping select memory
       allocations. -->
  <!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
  <!-- Ensure all image data is fully flushed and synchronized to disk. -->
  <!-- <policy domain="cache" name="synchronize" value="true"/> -->
  <!-- Replace passphrase for secure distributed processing -->
  <!-- <policy domain="cache" name="shared-secret" value="secret-passphrase" stealth="true"/> -->
  <!-- Do not permit any delegates to execute. -->
  <!-- <policy domain="delegate" rights="none" pattern="*"/> -->
  <!-- Do not permit any image filters to load. -->
  <!-- <policy domain="filter" rights="none" pattern="*"/> -->
  <!-- Don't read/write from/to stdin/stdout. -->
  <!-- <policy domain="path" rights="none" pattern="-"/> -->
  <!-- don't read sensitive paths. -->
  <!-- <policy domain="path" rights="none" pattern="/etc/*"/> -->
  <!-- Indirect reads are not permitted. -->
  <policy domain="path" rights="none" pattern="@*"/>
  <!-- These image types are security risks on read, but write is fine -->
  <!-- <policy domain="module" rights="write" pattern="{MSL,MVG,PS,SVG,URL,XPS}"/> -->
  <!-- This policy sets the number of times to replace content of certain
       memory buffers and temporary files before they are freed or deleted. -->
  <!-- <policy domain="system" name="shred" value="1"/> -->
  <!-- Enable the initialization of buffers with zeros, resulting in a minor
       performance penalty but with improved security. -->
  <!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
  <!-- Set the maximum amount of memory in bytes that are permitted for
       allocation requests. -->
  <!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
  <policy domain="delegate" rights="none" pattern="URL" />
  <policy domain="delegate" rights="none" pattern="HTTPS" />
  <policy domain="delegate" rights="none" pattern="HTTP" />
</policymap>