Deletion of directory /var/lib/incus/storage-pools/tank/virtual-machines/<machine> fails with error

[LarsKollstedt]

Is there an existing issue for this?

• There is no existing issue for this bug

Is this happening on an up to date version of Incus?

• This is happening on a supported version of Incus

Incus system details

root@incus1:~# incus info
config:
  cluster.https_address: incus1.man-da.de:8443
  core.https_address: :8443
  core.metrics_authentication: "false"
  core.remote_token_expiry: 1y
  instances.placement.scriptlet: |
    def instance_placement(request, candidate_members):
        config = request.config
        group = config.get("user.ha_group")
        if group == None:
            log_info("Using default placement strategy.")
            return

        log_info(
            "Determining placement of instance '{}' for high-availability group '{}'".format(
                request.name, group
            )
        )

        # Get all instances in the same project
        instances = get_instances(location="", project=request.project)

        # Track which cluster members already host a group instance
        unavailable_member_names = []
        for instance in instances:
            if instance.status != "Running":
                continue
            if instance.config.get("user.ha_group") == group:
                unavailable_member_names.append(instance.location)

        log_info(
            "Cluster members hosting resources in high-availability group '{}': {}".format(
                group, ", ".join(unavailable_member_names)
            )
        )

        available_members = [member.server_name for member in candidate_members if member.server_name not in unavailable_member_names]

        if not available_members:
            log_warn(
                "No available cluster members for high-availability group '{}' criteria. Falling back to default placement strategy.".format(
                    group
                )
            )
            return

        log_info(
            "Available cluster members for high-availability group '{}': {}".format(group, ", ".join(available_members))
        )

        # Choose the eligible member with the fewest instances
        def instance_count(name):
            return get_instances_count(location=name, project=request.project, pending=True)

        target_member = min(available_members, key=instance_count)
        set_target(target_member)
        log_info(
            "Placing instance '{instance}' on '{member}, because it has the lowest number of running resources.".format(
                instance=request.name, member=target_member
            )
        )
  logging.loki.lifecycle.types: instance
  logging.loki.target.address: https://loki.man-da.de
  logging.loki.target.type: loki
  logging.loki.types: lifecycle,network-acl
api_extensions_count: 529
api_status: stable
api_version: "1.0"
auth: trusted
public: false
auth_methods:
  - tls
auth_user_name: root
auth_user_method: unix
environment:
  addresses:
    - 82.195.93.133:8443
    - '[2001:41b8:41:41:c6cb:e1ff:fecb:4714]:8443'
    - 82.195.92.225:8443
    - '[2001:41b8:41:44:ac11:3fff:feab:d414]:8443'
  architectures:
    - x86_64
    - i686
  certificate: ""
  certificate_fingerprint: 81ffab613f1d1e8f03dedd5914c40fc4944a2e60425fe79a076408448947671b
  driver: lxc | qemu
  driver_version: 7.0.0 | 11.0.2
  firewall: nftables
  kernel: Linux
  kernel_architecture: x86_64
  kernel_features: {}
  kernel_version: 6.12.95+deb13-amd64
  lxc_features: {}
  os_name: Debian GNU/Linux
  os_version: "13"
  project: default
  server: incus
  server_clustered: true
  server_event_mode: full-mesh
  server_name: incus1.man-da.de
  server_pid: 2736573
  server_version: "7.2"
  storage: zfs
  storage_version: 2.3.2-2
  storage_supported_drivers:
    - name: dir
      version: "1"
      remote: false
    - name: lvm
      version: 2.03.31(2) (2025-02-27) / 1.02.205 (2025-02-27) / 4.48.0
      remote: false
    - name: truenas
      version: 0.7.7
      remote: true
    - name: zfs
      version: 2.3.2-2
      remote: false

Instance details

root@incus1:~# incus config show static
architecture: x86_64
config:
  image.architecture: amd64
  image.description: Debian trixie amd64 (20251126_05:24)
  image.os: Debian
  image.release: trixie
  image.serial: '20251126_05:24'
  image.type: disk-kvm.img
  image.variant: default
  volatile.base_image: a45641b6fbf0593f8e2aa080c924ff938d289d856c5dfdeeaba9694052ea9db6
  volatile.cloud-init.instance-id: d71fd51c-08e9-43ca-8450-beff9db7438f
  volatile.enp5s0.host_name: tapab9544f8
  volatile.enp5s0.hwaddr: 10:66:6a:17:7e:0b
  volatile.evacuate.origin: incus1.man-da.de
  volatile.last_state.agent: STARTED
  volatile.last_state.power: RUNNING
  volatile.last_state.ready: "false"
  volatile.uuid: d37cff6d-6b8f-4eb0-859c-c9f0c8d6d574
  volatile.uuid.generation: d37cff6d-6b8f-4eb0-859c-c9f0c8d6d574
  volatile.vm.boot_state: '{"version":1,"cpu_type":"kvm64,hv_relaxed,hv_vpindex,hv_runtime,hv_time,hv_synic,hv_stimer,hv_tlbflush,hv_ipi,hv_frequencies,hv_stimer_direct,hv_xmm_input,hv_tlbflush_ext,avx512bw,avx512f,rdpid,ssse3,xsaveopt,vaes,fsrm,pdpe1gb,pclmulqdq,avic,vgif,bmi1,succor,avx512vl,osvw,topoext,stibp,smep,flushbyasid,avx512vbmi,avx512ifma,f16c,skinit,perfctr_nb,ibs,smap,ht,extapic,erms,clflushopt,decodeassists,svm,ssbd,bmi2,adx,xsaveerptr,sse4a,vmcb_clean,pfthreshold,xsave,3dnowprefetch,wbnoinvd,svm_lock,aes,ibrs,ibpb,umip,vpclmulqdq,rdtscp,invpcid,fma,misalignsse,avx2,arat,fsgsbase,avx512dq,tsc_scale,la57,lahf_lm,cmp_legacy,perfctr_core,rdrand,npt,lbrv,sse4_1,fxsr_opt,wdt,xgetbv1,pcid,tce,clzero,gfni,monitor,rdseed,vnmi,pku,movbe,popcnt,xsaves,avx,clwb,mmxext,sse4_2,xsavec,nrip_save,abm,avx512cd","cpu_topology":{"sockets":1,"cores":2,"threads":1,"explicit":false},"machine_type":"pc-q35-11.0","memory_topology":{"base":2147483648,"max":274877906944,"extra":null}}'
  volatile.vm.rtc_adjustment: "0"
  volatile.vm.rtc_offset: "0"
  volatile.vsock_id: "1578608988"
devices:
  enp5s0:
    network: manda-vlan522
    type: nic
ephemeral: false
profiles:
  - vm
  - cpu2
  - ram2
  - disk32
stateful: false
description: ""

Instance log

root@incus1:~# incus info --show-log static
Name: static
Description: 
Status: RUNNING
Type: virtual-machine
Architecture: x86_64
Location: incus4.man-da.de
PID: 2953046
Created: 2025/11/26 15:17 CET
Last Used: 2026/07/21 12:06 CEST
Started: 2026/07/21 12:06 CEST

Operating System:
  OS: Debian GNU/Linux
  OS Version: 13
  Kernel Version: 6.12.95+deb13-amd64
  Hostname: static.man-da.de
  FQDN: localhost

Resources:
  Processes: 124
  Disk usage:
    root: 7.16GiB
  CPU usage:
    CPU usage (in seconds): 109
  Memory usage:
    Memory (current): 600.89MiB
  Network usage:
    enp5s0:
      Type: broadcast
      State: UP
      Host interface: tapab9544f8
      MAC address: 10:66:6a:17:7e:0b
      MTU: 1500
      Bytes received: 4.20MB
      Bytes sent: 9.97MB
      Packets received: 34923
      Packets sent: 33928
      IP addresses:
        inet:  82.195.66.235/27 (global)
        inet6: 2001:41b8:0:1:1266:6aff:fe17:7e0b/64 (global)
        inet6: fe80::1266:6aff:fe17:7e0b/64 (link)
    lo:
      Type: loopback
      State: UP
      MTU: 65536
      Bytes received: 2.50MB
      Bytes sent: 2.50MB
      Packets received: 5585
      Packets sent: 5585
      IP addresses:
        inet:  127.0.0.1/8 (local)
        inet6: ::1/128 (local)

Log (qemu.log):


root@incus1:~#

Current behavior

Evacuation fails with:

# incus cluster evacuate "incus1.man-da.de" --debug --action auto --force
time="2026-07-21T12:06:06+02:00" level=debug msg="Connecting to a local Incus over a Unix socket"
time="2026-07-21T12:06:06+02:00" level=debug msg="Sending request to Incus" etag= method=GET url="http://unix.socket/1.0"
time="2026-07-21T12:06:06+02:00" level=debug msg="Got response struct from Incus"
time="2026-07-21T12:06:06+02:00" level=debug msg="\n\t{\n\t\t\"config\": {\n\t\t\t\"cluster.https_address\": \"incus1.man-da.de:8443\",\n\t\t\t\"core.https_address\": \":8443\",\n\t\t\t\"core.metrics_authentication\": \"false\",\n\t\t\t\"core.remote_token_expiry\": \"1y\",\n\t\t\t\"instances.placement.scriptlet\": \"def instance_placement(request, candidate_members):\\n    config = request.config\\n    group = config.get(\\\"user.ha_group\\\")\\n    if group == None:\\n        log_info(\\\"Using default placement strategy.\\\")\\n        return\\n\\n    log_info(\\n        \\\"Determining placement of instance '{}' for high-availability group '{}'\\\".format(\\n            request.name, group\\n        )\\n    )\\n\\n    # Get all instances in the same project\\n    instances = get_instances(location=\\\"\\\", project=request.project)\\n\\n    # Track which cluster members already host a group instance\\n    unavailable_member_names = []\\n    for instance in instances:\\n        if instance.status != \\\"Running\\\":\\n            continue\\n        if instance.config.get(\\\"user.ha_group\\\") == group:\\n            unavailable_member_names.append(instance.location)\\n\\n    log_info(\\n        \\\"Cluster members hosting resources in high-availability group '{}': {}\\\".format(\\n            group, \\\", \\\".join(unavailable_member_names)\\n        )\\n    )\\n\\n    available_members = [member.server_name for member in candidate_members if member.server_name not in unavailable_member_names]\\n\\n    if not available_members:\\n        log_warn(\\n            \\\"No available cluster members for high-availability group '{}' criteria. Falling back to default placement strategy.\\\".format(\\n                group\\n            )\\n        )\\n        return\\n\\n    log_info(\\n        \\\"Available cluster members for high-availability group '{}': {}\\\".format(group, \\\", \\\".join(available_members))\\n    )\\n\\n    # Choose the eligible member with the fewest instances\\n    def instance_count(name):\\n        return get_instances_count(location=name, project=request.project, pending=True)\\n\\n    target_member = min(available_members, key=instance_count)\\n    set_target(target_member)\\n    log_info(\\n        \\\"Placing instance '{instance}' on '{member}, because it has the lowest number of running resources.\\\".format(\\n            instance=request.name, member=target_member\\n        )\\n    )\\n\",\n\t\t\t\"logging.loki.lifecycle.types\": \"instance\",\n\t\t\t\"logging.loki.target.address\": \"https://loki.man-da.de\",\n\t\t\t\"logging.loki.target.type\": \"loki\",\n\t\t\t\"logging.loki.types\": \"lifecycle,network-acl\"\n\t\t},\n\t\t\"api_extensions\": [\n\t\t\t\"storage_zfs_remove_snapshots\",\n\t\t\t\"container_host_shutdown_timeout\",\n\t\t\t\"container_stop_priority\",\n\t\t\t\"container_syscall_filtering\",\n\t\t\t\"auth_pki\",\n\t\t\t\"container_last_used_at\",\n\t\t\t\"etag\",\n\t\t\t\"patch\",\n\t\t\t\"usb_devices\",\n\t\t\t\"https_allowed_credentials\",\n\t\t\t\"image_compression_algorithm\",\n\t\t\t\"directory_manipulation\",\n\t\t\t\"container_cpu_time\",\n\t\t\t\"storage_zfs_use_refquota\",\n\t\t\t\"storage_lvm_mount_options\",\n\t\t\t\"network\",\n\t\t\t\"profile_usedby\",\n\t\t\t\"container_push\",\n\t\t\t\"container_exec_recording\",\n\t\t\t\"certificate_update\",\n\t\t\t\"container_exec_signal_handling\",\n\t\t\t\"gpu_devices\",\n\t\t\t\"container_image_properties\",\n\t\t\t\"migration_progress\",\n\t\t\t\"id_map\",\n\t\t\t\"network_firewall_filtering\",\n\t\t\t\"network_routes\",\n\t\t\t\"storage\",\n\t\t\t\"file_delete\",\n\t\t\t\"file_append\",\n\t\t\t\"network_dhcp_expiry\",\n\t\t\t\"storage_lvm_vg_rename\",\n\t\t\t\"storage_lvm_thinpool_rename\",\n\t\t\t\"network_vlan\",\n\t\t\t\"image_create_aliases\",\n\t\t\t\"container_stateless_copy\",\n\t\t\t\"container_only_migration\",\n\t\t\t\"storage_zfs_clone_copy\",\n\t\t\t\"unix_device_rename\",\n\t\t\t\"storage_lvm_use_thinpool\",\n\t\t\t\"storage_rsync_bwlimit\",\n\t\t\t\"network_vxlan_interface\",\n\t\t\t\"storage_btrfs_mount_options\",\n\t\t\t\"entity_description\",\n\t\t\t\"image_force_refresh\",\n\t\t\t\"storage_lvm_lv_resizing\",\n\t\t\t\"id_map_base\",\n\t\t\t\"file_symlinks\",\n\t\t\t\"container_push_target\",\n\t\t\t\"network_vlan_physical\",\n\t\t\t\"storage_images_delete\",\n\t\t\t\"container_edit_metadata\",\n\t\t\t\"container_snapshot_stateful_migration\",\n\t\t\t\"storage_driver_ceph\",\n\t\t\t\"storage_ceph_user_name\",\n\t\t\t\"resource_limits\",\n\t\t\t\"storage_volatile_initial_source\",\n\t\t\t\"storage_ceph_force_osd_reuse\",\n\t\t\t\"storage_block_filesystem_btrfs\",\n\t\t\t\"resources\",\n\t\t\t\"kernel_limits\",\n\t\t\t\"storage_api_volume_rename\",\n\t\t\t\"network_sriov\",\n\t\t\t\"console\",\n\t\t\t\"restrict_dev_incus\",\n\t\t\t\"migration_pre_copy\",\n\t\t\t\"infiniband\",\n\t\t\t\"dev_incus_events\",\n\t\t\t\"proxy\",\n\t\t\t\"network_dhcp_gateway\",\n\t\t\t\"file_get_symlink\",\n\t\t\t\"network_leases\",\n\t\t\t\"unix_device_hotplug\",\n\t\t\t\"storage_api_local_volume_handling\",\n\t\t\t\"operation_description\",\n\t\t\t\"clustering\",\n\t\t\t\"event_lifecycle\",\n\t\t\t\"storage_api_remote_volume_handling\",\n\t\t\t\"nvidia_runtime\",\n\t\t\t\"container_mount_propagation\",\n\t\t\t\"container_backup\",\n\t\t\t\"dev_incus_images\",\n\t\t\t\"container_local_cross_pool_handling\",\n\t\t\t\"proxy_unix\",\n\t\t\t\"proxy_udp\",\n\t\t\t\"clustering_join\",\n\t\t\t\"proxy_tcp_udp_multi_port_handling\",\n\t\t\t\"network_state\",\n\t\t\t\"proxy_unix_dac_properties\",\n\t\t\t\"container_protection_delete\",\n\t\t\t\"unix_priv_drop\",\n\t\t\t\"pprof_http\",\n\t\t\t\"proxy_haproxy_protocol\",\n\t\t\t\"network_hwaddr\",\n\t\t\t\"proxy_nat\",\n\t\t\t\"network_nat_order\",\n\t\t\t\"container_full\",\n\t\t\t\"backup_compression\",\n\t\t\t\"nvidia_runtime_config\",\n\t\t\t\"storage_api_volume_snapshots\",\n\t\t\t\"storage_unmapped\",\n\t\t\t\"projects\",\n\t\t\t\"network_vxlan_ttl\",\n\t\t\t\"container_incremental_copy\",\n\t\t\t\"usb_optional_vendorid\",\n\t\t\t\"snapshot_scheduling\",\n\t\t\t\"snapshot_schedule_aliases\",\n\t\t\t\"container_copy_project\",\n\t\t\t\"clustering_server_address\",\n\t\t\t\"clustering_image_replication\",\n\t\t\t\"container_protection_shift\",\n\t\t\t\"snapshot_expiry\",\n\t\t\t\"container_backup_override_pool\",\n\t\t\t\"snapshot_expiry_creation\",\n\t\t\t\"network_leases_location\",\n\t\t\t\"resources_cpu_socket\",\n\t\t\t\"resources_gpu\",\n\t\t\t\"resources_numa\",\n\t\t\t\"kernel_features\",\n\t\t\t\"id_map_current\",\n\t\t\t\"event_location\",\n\t\t\t\"storage_api_remote_volume_snapshots\",\n\t\t\t\"network_nat_address\",\n\t\t\t\"container_nic_routes\",\n\t\t\t\"cluster_internal_copy\",\n\t\t\t\"seccomp_notify\",\n\t\t\t\"lxc_features\",\n\t\t\t\"container_nic_ipvlan\",\n\t\t\t\"network_vlan_sriov\",\n\t\t\t\"storage_cephfs\",\n\t\t\t\"container_nic_ipfilter\",\n\t\t\t\"resources_v2\",\n\t\t\t\"container_exec_user_group_cwd\",\n\t\t\t\"container_syscall_intercept\",\n\t\t\t\"container_disk_shift\",\n\t\t\t\"storage_shifted\",\n\t\t\t\"resources_infiniband\",\n\t\t\t\"daemon_storage\",\n\t\t\t\"instances\",\n\t\t\t\"image_types\",\n\t\t\t\"resources_disk_sata\",\n\t\t\t\"clustering_roles\",\n\t\t\t\"images_expiry\",\n\t\t\t\"resources_network_firmware\",\n\t\t\t\"backup_compression_algorithm\",\n\t\t\t\"ceph_data_pool_name\",\n\t\t\t\"container_syscall_intercept_mount\",\n\t\t\t\"compression_squashfs\",\n\t\t\t\"container_raw_mount\",\n\t\t\t\"container_nic_routed\",\n\t\t\t\"container_syscall_intercept_mount_fuse\",\n\t\t\t\"container_disk_ceph\",\n\t\t\t\"virtual-machines\",\n\t\t\t\"image_profiles\",\n\t\t\t\"clustering_architecture\",\n\t\t\t\"resources_disk_id\",\n\t\t\t\"storage_lvm_stripes\",\n\t\t\t\"vm_boot_priority\",\n\t\t\t\"unix_hotplug_devices\",\n\t\t\t\"api_filtering\",\n\t\t\t\"instance_nic_network\",\n\t\t\t\"clustering_sizing\",\n\t\t\t\"firewall_driver\",\n\t\t\t\"projects_limits\",\n\t\t\t\"container_syscall_intercept_hugetlbfs\",\n\t\t\t\"limits_hugepages\",\n\t\t\t\"container_nic_routed_gateway\",\n\t\t\t\"projects_restrictions\",\n\t\t\t\"custom_volume_snapshot_expiry\",\n\t\t\t\"volume_snapshot_scheduling\",\n\t\t\t\"trust_ca_certificates\",\n\t\t\t\"snapshot_disk_usage\",\n\t\t\t\"clustering_edit_roles\",\n\t\t\t\"container_nic_routed_host_address\",\n\t\t\t\"container_nic_ipvlan_gateway\",\n\t\t\t\"resources_usb_pci\",\n\t\t\t\"resources_cpu_threads_numa\",\n\t\t\t\"resources_cpu_core_die\",\n\t\t\t\"api_os\",\n\t\t\t\"container_nic_routed_host_table\",\n\t\t\t\"container_nic_ipvlan_host_table\",\n\t\t\t\"container_nic_ipvlan_mode\",\n\t\t\t\"resources_system\",\n\t\t\t\"images_push_relay\",\n\t\t\t\"network_dns_search\",\n\t\t\t\"container_nic_routed_limits\",\n\t\t\t\"instance_nic_bridged_vlan\",\n\t\t\t\"network_state_bond_bridge\",\n\t\t\t\"usedby_consistency\",\n\t\t\t\"custom_block_volumes\",\n\t\t\t\"clustering_failure_domains\",\n\t\t\t\"resources_gpu_mdev\",\n\t\t\t\"console_vga_type\",\n\t\t\t\"projects_limits_disk\",\n\t\t\t\"network_type_macvlan\",\n\t\t\t\"network_type_sriov\",\n\t\t\t\"container_syscall_intercept_bpf_devices\",\n\t\t\t\"network_type_ovn\",\n\t\t\t\"projects_networks\",\n\t\t\t\"projects_networks_restricted_uplinks\",\n\t\t\t\"custom_volume_backup\",\n\t\t\t\"backup_override_name\",\n\t\t\t\"storage_rsync_compression\",\n\t\t\t\"network_type_physical\",\n\t\t\t\"network_ovn_external_subnets\",\n\t\t\t\"network_ovn_nat\",\n\t\t\t\"network_ovn_external_routes_remove\",\n\t\t\t\"tpm_device_type\",\n\t\t\t\"storage_zfs_clone_copy_rebase\",\n\t\t\t\"gpu_mdev\",\n\t\t\t\"resources_pci_iommu\",\n\t\t\t\"resources_network_usb\",\n\t\t\t\"resources_disk_address\",\n\t\t\t\"network_physical_ovn_ingress_mode\",\n\t\t\t\"network_ovn_dhcp\",\n\t\t\t\"network_physical_routes_anycast\",\n\t\t\t\"projects_limits_instances\",\n\t\t\t\"network_state_vlan\",\n\t\t\t\"instance_nic_bridged_port_isolation\",\n\t\t\t\"instance_bulk_state_change\",\n\t\t\t\"network_gvrp\",\n\t\t\t\"instance_pool_move\",\n\t\t\t\"gpu_sriov\",\n\t\t\t\"pci_device_type\",\n\t\t\t\"storage_volume_state\",\n\t\t\t\"network_acl\",\n\t\t\t\"migration_stateful\",\n\t\t\t\"disk_state_quota\",\n\t\t\t\"storage_ceph_features\",\n\t\t\t\"projects_compression\",\n\t\t\t\"projects_images_remote_cache_expiry\",\n\t\t\t\"certificate_project\",\n\t\t\t\"network_ovn_acl\",\n\t\t\t\"projects_images_auto_update\",\n\t\t\t\"projects_restricted_cluster_target\",\n\t\t\t\"images_default_architecture\",\n\t\t\t\"network_ovn_acl_defaults\",\n\t\t\t\"gpu_mig\",\n\t\t\t\"project_usage\",\n\t\t\t\"network_bridge_acl\",\n\t\t\t\"warnings\",\n\t\t\t\"projects_restricted_backups_and_snapshots\",\n\t\t\t\"clustering_join_token\",\n\t\t\t\"clustering_description\",\n\t\t\t\"server_trusted_proxy\",\n\t\t\t\"clustering_update_cert\",\n\t\t\t\"storage_api_project\",\n\t\t\t\"server_instance_driver_operational\",\n\t\t\t\"server_supported_storage_drivers\",\n\t\t\t\"event_lifecycle_requestor_address\",\n\t\t\t\"resources_gpu_usb\",\n\t\t\t\"clustering_evacuation\",\n\t\t\t\"network_ovn_nat_address\",\n\t\t\t\"network_bgp\",\n\t\t\t\"network_forward\",\n\t\t\t\"custom_volume_refresh\",\n\t\t\t\"network_counters_errors_dropped\",\n\t\t\t\"metrics\",\n\t\t\t\"image_source_project\",\n\t\t\t\"clustering_config\",\n\t\t\t\"network_peer\",\n\t\t\t\"linux_sysctl\",\n\t\t\t\"network_dns\",\n\t\t\t\"ovn_nic_acceleration\",\n\t\t\t\"certificate_self_renewal\",\n\t\t\t\"instance_project_move\",\n\t\t\t\"storage_volume_project_move\",\n\t\t\t\"cloud_init\",\n\t\t\t\"network_dns_nat\",\n\t\t\t\"database_leader\",\n\t\t\t\"instance_all_projects\",\n\t\t\t\"clustering_groups\",\n\t\t\t\"ceph_rbd_du\",\n\t\t\t\"instance_get_full\",\n\t\t\t\"qemu_metrics\",\n\t\t\t\"gpu_mig_uuid\",\n\t\t\t\"event_project\",\n\t\t\t\"clustering_evacuation_live\",\n\t\t\t\"instance_allow_inconsistent_copy\",\n\t\t\t\"network_state_ovn\",\n\t\t\t\"storage_volume_api_filtering\",\n\t\t\t\"image_restrictions\",\n\t\t\t\"storage_zfs_export\",\n\t\t\t\"network_dns_records\",\n\t\t\t\"storage_zfs_reserve_space\",\n\t\t\t\"network_acl_log\",\n\t\t\t\"storage_zfs_blocksize\",\n\t\t\t\"metrics_cpu_seconds\",\n\t\t\t\"instance_snapshot_never\",\n\t\t\t\"certificate_token\",\n\t\t\t\"instance_nic_routed_neighbor_probe\",\n\t\t\t\"event_hub\",\n\t\t\t\"agent_nic_config\",\n\t\t\t\"projects_restricted_intercept\",\n\t\t\t\"metrics_authentication\",\n\t\t\t\"images_target_project\",\n\t\t\t\"images_all_projects\",\n\t\t\t\"cluster_migration_inconsistent_copy\",\n\t\t\t\"cluster_ovn_chassis\",\n\t\t\t\"container_syscall_intercept_sched_setscheduler\",\n\t\t\t\"storage_lvm_thinpool_metadata_size\",\n\t\t\t\"storage_volume_state_total\",\n\t\t\t\"instance_file_head\",\n\t\t\t\"instances_nic_host_name\",\n\t\t\t\"image_copy_profile\",\n\t\t\t\"container_syscall_intercept_sysinfo\",\n\t\t\t\"clustering_evacuation_mode\",\n\t\t\t\"resources_pci_vpd\",\n\t\t\t\"qemu_raw_conf\",\n\t\t\t\"storage_cephfs_fscache\",\n\t\t\t\"network_load_balancer\",\n\t\t\t\"vsock_api\",\n\t\t\t\"instance_ready_state\",\n\t\t\t\"network_bgp_holdtime\",\n\t\t\t\"storage_volumes_all_projects\",\n\t\t\t\"metrics_memory_oom_total\",\n\t\t\t\"storage_buckets\",\n\t\t\t\"storage_buckets_create_credentials\",\n\t\t\t\"metrics_cpu_effective_total\",\n\t\t\t\"projects_networks_restricted_access\",\n\t\t\t\"storage_buckets_local\",\n\t\t\t\"loki\",\n\t\t\t\"acme\",\n\t\t\t\"internal_metrics\",\n\t\t\t\"cluster_join_token_expiry\",\n\t\t\t\"remote_token_expiry\",\n\t\t\t\"init_preseed\",\n\t\t\t\"storage_volumes_created_at\",\n\t\t\t\"cpu_hotplug\",\n\t\t\t\"projects_networks_zones\",\n\t\t\t\"network_txqueuelen\",\n\t\t\t\"cluster_member_state\",\n\t\t\t\"instances_placement_scriptlet\",\n\t\t\t\"storage_pool_source_wipe\",\n\t\t\t\"zfs_block_mode\",\n\t\t\t\"instance_generation_id\",\n\t\t\t\"disk_io_cache\",\n\t\t\t\"amd_sev\",\n\t\t\t\"storage_pool_loop_resize\",\n\t\t\t\"migration_vm_live\",\n\t\t\t\"ovn_nic_nesting\",\n\t\t\t\"oidc\",\n\t\t\t\"network_ovn_l3only\",\n\t\t\t\"ovn_nic_acceleration_vdpa\",\n\t\t\t\"cluster_healing\",\n\t\t\t\"instances_state_total\",\n\t\t\t\"auth_user\",\n\t\t\t\"security_csm\",\n\t\t\t\"instances_rebuild\",\n\t\t\t\"numa_cpu_placement\",\n\t\t\t\"custom_volume_iso\",\n\t\t\t\"network_allocations\",\n\t\t\t\"zfs_delegate\",\n\t\t\t\"storage_api_remote_volume_snapshot_copy\",\n\t\t\t\"operations_get_query_all_projects\",\n\t\t\t\"metadata_configuration\",\n\t\t\t\"syslog_socket\",\n\t\t\t\"event_lifecycle_name_and_project\",\n\t\t\t\"instances_nic_limits_priority\",\n\t\t\t\"disk_initial_volume_configuration\",\n\t\t\t\"operation_wait\",\n\t\t\t\"image_restriction_privileged\",\n\t\t\t\"cluster_internal_custom_volume_copy\",\n\t\t\t\"disk_io_bus\",\n\t\t\t\"storage_cephfs_create_missing\",\n\t\t\t\"instance_move_config\",\n\t\t\t\"ovn_ssl_config\",\n\t\t\t\"certificate_description\",\n\t\t\t\"disk_io_bus_virtio_blk\",\n\t\t\t\"loki_config_instance\",\n\t\t\t\"instance_create_start\",\n\t\t\t\"clustering_evacuation_stop_options\",\n\t\t\t\"boot_host_shutdown_action\",\n\t\t\t\"agent_config_drive\",\n\t\t\t\"network_state_ovn_lr\",\n\t\t\t\"image_template_permissions\",\n\t\t\t\"storage_bucket_backup\",\n\t\t\t\"storage_lvm_cluster\",\n\t\t\t\"shared_custom_block_volumes\",\n\t\t\t\"auth_tls_jwt\",\n\t\t\t\"oidc_claim\",\n\t\t\t\"device_usb_serial\",\n\t\t\t\"numa_cpu_balanced\",\n\t\t\t\"image_restriction_nesting\",\n\t\t\t\"network_integrations\",\n\t\t\t\"instance_memory_swap_bytes\",\n\t\t\t\"network_bridge_external_create\",\n\t\t\t\"network_zones_all_projects\",\n\t\t\t\"storage_zfs_vdev\",\n\t\t\t\"container_migration_stateful\",\n\t\t\t\"profiles_all_projects\",\n\t\t\t\"instances_scriptlet_get_instances\",\n\t\t\t\"instances_scriptlet_get_cluster_members\",\n\t\t\t\"instances_scriptlet_get_project\",\n\t\t\t\"network_acl_stateless\",\n\t\t\t\"instance_state_started_at\",\n\t\t\t\"networks_all_projects\",\n\t\t\t\"network_acls_all_projects\",\n\t\t\t\"storage_buckets_all_projects\",\n\t\t\t\"resources_load\",\n\t\t\t\"instance_access\",\n\t\t\t\"project_access\",\n\t\t\t\"projects_force_delete\",\n\t\t\t\"resources_cpu_flags\",\n\t\t\t\"disk_io_bus_cache_filesystem\",\n\t\t\t\"instance_oci\",\n\t\t\t\"clustering_groups_config\",\n\t\t\t\"instances_lxcfs_per_instance\",\n\t\t\t\"clustering_groups_vm_cpu_definition\",\n\t\t\t\"disk_volume_subpath\",\n\t\t\t\"projects_limits_disk_pool\",\n\t\t\t\"network_ovn_isolated\",\n\t\t\t\"qemu_raw_qmp\",\n\t\t\t\"network_load_balancer_health_check\",\n\t\t\t\"oidc_scopes\",\n\t\t\t\"network_integrations_peer_name\",\n\t\t\t\"qemu_scriptlet\",\n\t\t\t\"instance_auto_restart\",\n\t\t\t\"storage_lvm_metadatasize\",\n\t\t\t\"ovn_nic_promiscuous\",\n\t\t\t\"ovn_nic_ip_address_none\",\n\t\t\t\"instances_state_os_info\",\n\t\t\t\"network_load_balancer_state\",\n\t\t\t\"instance_nic_macvlan_mode\",\n\t\t\t\"storage_lvm_cluster_create\",\n\t\t\t\"network_ovn_external_interfaces\",\n\t\t\t\"instances_scriptlet_get_instances_count\",\n\t\t\t\"cluster_rebalance\",\n\t\t\t\"custom_volume_refresh_exclude_older_snapshots\",\n\t\t\t\"storage_initial_owner\",\n\t\t\t\"storage_live_migration\",\n\t\t\t\"instance_console_screenshot\",\n\t\t\t\"image_import_alias\",\n\t\t\t\"authorization_scriptlet\",\n\t\t\t\"console_force\",\n\t\t\t\"network_ovn_state_addresses\",\n\t\t\t\"network_bridge_acl_devices\",\n\t\t\t\"instance_debug_memory\",\n\t\t\t\"init_preseed_storage_volumes\",\n\t\t\t\"init_preseed_profile_project\",\n\t\t\t\"instance_nic_routed_host_address\",\n\t\t\t\"instance_smbios11\",\n\t\t\t\"api_filtering_extended\",\n\t\t\t\"acme_dns01\",\n\t\t\t\"security_iommu\",\n\t\t\t\"network_ipv4_dhcp_routes\",\n\t\t\t\"network_state_ovn_ls\",\n\t\t\t\"network_dns_nameservers\",\n\t\t\t\"acme_http01_port\",\n\t\t\t\"network_ovn_ipv4_dhcp_expiry\",\n\t\t\t\"instance_state_cpu_time\",\n\t\t\t\"network_io_bus\",\n\t\t\t\"disk_io_bus_usb\",\n\t\t\t\"storage_driver_linstor\",\n\t\t\t\"instance_oci_entrypoint\",\n\t\t\t\"network_address_set\",\n\t\t\t\"server_logging\",\n\t\t\t\"network_forward_snat\",\n\t\t\t\"memory_hotplug\",\n\t\t\t\"instance_nic_routed_host_tables\",\n\t\t\t\"instance_publish_split\",\n\t\t\t\"init_preseed_certificates\",\n\t\t\t\"custom_volume_sftp\",\n\t\t\t\"network_ovn_external_nic_address\",\n\t\t\t\"network_physical_gateway_hwaddr\",\n\t\t\t\"backup_s3_upload\",\n\t\t\t\"snapshot_manual_expiry\",\n\t\t\t\"resources_cpu_address_sizes\",\n\t\t\t\"disk_attached\",\n\t\t\t\"limits_memory_hotplug\",\n\t\t\t\"disk_wwn\",\n\t\t\t\"server_logging_webhook\",\n\t\t\t\"storage_driver_truenas\",\n\t\t\t\"container_disk_tmpfs\",\n\t\t\t\"instance_limits_oom\",\n\t\t\t\"backup_override_config\",\n\t\t\t\"network_ovn_tunnels\",\n\t\t\t\"init_preseed_cluster_groups\",\n\t\t\t\"usb_attached\",\n\t\t\t\"backup_iso\",\n\t\t\t\"instance_systemd_credentials\",\n\t\t\t\"cluster_group_usedby\",\n\t\t\t\"bpf_token_delegation\",\n\t\t\t\"file_storage_volume\",\n\t\t\t\"network_hwaddr_pattern\",\n\t\t\t\"storage_volume_full\",\n\t\t\t\"storage_bucket_full\",\n\t\t\t\"device_pci_firmware\",\n\t\t\t\"resources_serial\",\n\t\t\t\"ovn_nic_limits\",\n\t\t\t\"storage_lvmcluster_qcow2\",\n\t\t\t\"oidc_allowed_subnets\",\n\t\t\t\"file_delete_force\",\n\t\t\t\"nic_sriov_select_ext\",\n\t\t\t\"network_zones_dns_contact\",\n\t\t\t\"nic_attached_connected\",\n\t\t\t\"nic_sriov_security_trusted\",\n\t\t\t\"direct_backup\",\n\t\t\t\"instance_snapshot_disk_only_restore\",\n\t\t\t\"unix_hotplug_pci\",\n\t\t\t\"cluster_evacuating_restoring\",\n\t\t\t\"projects_restricted_image_servers\",\n\t\t\t\"storage_lvmcluster_size\",\n\t\t\t\"authorization_scriptlet_cert\",\n\t\t\t\"lvmcluster_remove_snapshots\",\n\t\t\t\"daemon_storage_logs\",\n\t\t\t\"instances_debug_repair\",\n\t\t\t\"network_io_bus_ovn\",\n\t\t\t\"dependent\",\n\t\t\t\"metrics_project_resources\",\n\t\t\t\"storage_volume_nbd\",\n\t\t\t\"projects_restricted_storage_pool_access\",\n\t\t\t\"server_shutdown_action\",\n\t\t\t\"instances_placement_scriptlet_rebalance\",\n\t\t\t\"network_bridge_multicast_snooping\",\n\t\t\t\"storage_create_options\",\n\t\t\t\"instances_tpm_platform_cert\",\n\t\t\t\"linstor_raw\",\n\t\t\t\"network_address_set_ip_ranges\",\n\t\t\t\"storage_volumes_rebuild\",\n\t\t\t\"api_fragments\",\n\t\t\t\"instance_limits_cpu_topology\",\n\t\t\t\"instance_nbd\",\n\t\t\t\"network_bridge_bgp_instances\",\n\t\t\t\"core_https_allowed_websocket_origin\",\n\t\t\t\"storage_btrfs_compression\",\n\t\t\t\"oci_network_config\",\n\t\t\t\"infiniband_sriov_guid\",\n\t\t\t\"instance_selinux\"\n\t\t],\n\t\t\"api_status\": \"stable\",\n\t\t\"api_version\": \"1.0\",\n\t\t\"auth\": \"trusted\",\n\t\t\"public\": false,\n\t\t\"auth_methods\": [\n\t\t\t\"tls\"\n\t\t],\n\t\t\"auth_user_name\": \"root\",\n\t\t\"auth_user_method\": \"unix\",\n\t\t\"environment\": {\n\t\t\t\"addresses\": [\n\t\t\t\t\"82.195.93.133:8443\",\n\t\t\t\t\"[2001:41b8:41:41:c6cb:e1ff:fecb:4714]:8443\",\n\t\t\t\t\"82.195.92.225:8443\",\n\t\t\t\t\"[2001:41b8:41:44:ac11:3fff:feab:d414]:8443\"\n\t\t\t],\n\t\t\t\"architectures\": [\n\t\t\t\t\"x86_64\",\n\t\t\t\t\"i686\"\n\t\t\t],\n\t\t\t\"certificate\": \"-----BEGIN CERTIFICATE-----\\nMIID5jCCA2ygAwIBAgISBYFIzh4Iivy4CBpfdgzpQm07MAoGCCqGSM49BAMDMDMx\\nCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MQwwCgYDVQQDEwNZ\\nRTEwHhcNMjYwNjIxMTcwMjI5WhcNMjYwOTE5MTcwMjI4WjAbMRkwFwYDVQQDExBp\\nbmN1czEubWFuLWRhLmRlMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE5tnqU/ph\\nXhfU7QGwTJcoiBVoRZzULv6jOME31DItde1XdqdOfuZqaN/QtqjF9IwpcMQEX90q\\n12P1vW0NFDlweaOCAnYwggJyMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggr\\nBgEFBQcDATAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBTr9pXOpatureVJJGOiQ4so\\nSzFTVjAfBgNVHSMEGDAWgBS7IMpHC/7X5Zz5jwkqo4w3RbG82DAzBggrBgEFBQcB\\nAQQnMCUwIwYIKwYBBQUHMAKGF2h0dHA6Ly95ZTEuaS5sZW5jci5vcmcvMHQGA1Ud\\nEQRtMGuCD2luY3VzLm1hbi1kYS5kZYIQaW5jdXMxLm1hbi1kYS5kZYIQaW5jdXMy\\nLm1hbi1kYS5kZYIQaW5jdXMzLm1hbi1kYS5kZYIQaW5jdXM0Lm1hbi1kYS5kZYIQ\\naW5jdXM1Lm1hbi1kYS5kZTATBgNVHSAEDDAKMAgGBmeBDAECATAuBgNVHR8EJzAl\\nMCOgIaAfhh1odHRwOi8veWUxLmMubGVuY3Iub3JnLzYwLmNybDCCAQsGCisGAQQB\\n1nkCBAIEgfwEgfkA9wB1ANgJVTuUT3r/yBYZb5RPhauw+Pxeh1UmDxXRLnK7RUsU\\nAAABnutXyvkAAAQDAEYwRAIgAsAcxTPqWMLiBLKygWOZejteRaeYlU+Wp2ZFfjES\\nYF4CIB95dh+pgZUcoLEjtkr1RGUR2tENPflwWtXBC3EUzRFrAH4AbP5QGUOoXqkW\\nvFLRM+TcyR7xQRx9JYQg0XOAnhgY6zoAAAGe61fMHQAIAAAFABF6CC8EAwBHMEUC\\nIHHJs4VaMk9DGrFTzfVnsvymg/XSN8QA+k7VybE3JqXWAiEAwyXf7D9isgV+Nhty\\njungYRMo8Ep+4cWOB8V3p3Xe5eIwCgYIKoZIzj0EAwMDaAAwZQIwEEs2+EFI3QCG\\nyfvb2HVgzxq9CT5GTMqOQIAFUbLw9NajX8lpiCnI+MzsJpWes1xZAjEAgew6Ysd0\\nI+ecE0APfbQgi1q/kb/RfjLqHknxIo7EMOmYNvgeLnq4W5fQ7rUJM54n\\n-----END CERTIFICATE-----\\n\",\n\t\t\t\"certificate_fingerprint\": \"81ffab613f1d1e8f03dedd5914c40fc4944a2e60425fe79a076408448947671b\",\n\t\t\t\"driver\": \"lxc | qemu\",\n\t\t\t\"driver_version\": \"7.0.0 | 11.0.2\",\n\t\t\t\"firewall\": \"nftables\",\n\t\t\t\"kernel\": \"Linux\",\n\t\t\t\"kernel_architecture\": \"x86_64\",\n\t\t\t\"kernel_features\": {},\n\t\t\t\"kernel_version\": \"6.12.95+deb13-amd64\",\n\t\t\t\"lxc_features\": {},\n\t\t\t\"os_name\": \"Debian GNU/Linux\",\n\t\t\t\"os_version\": \"13\",\n\t\t\t\"project\": \"default\",\n\t\t\t\"server\": \"incus\",\n\t\t\t\"server_clustered\": true,\n\t\t\t\"server_event_mode\": \"full-mesh\",\n\t\t\t\"server_name\": \"incus1.man-da.de\",\n\t\t\t\"server_pid\": 2736573,\n\t\t\t\"server_version\": \"7.2\",\n\t\t\t\"storage\": \"zfs\",\n\t\t\t\"storage_version\": \"2.3.2-2\",\n\t\t\t\"storage_supported_drivers\": [\n\t\t\t\t{\n\t\t\t\t\t\"Name\": \"dir\",\n\t\t\t\t\t\"Version\": \"1\",\n\t\t\t\t\t\"Remote\": false\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"Name\": \"lvm\",\n\t\t\t\t\t\"Version\": \"2.03.31(2) (2025-02-27) / 1.02.205 (2025-02-27) / 4.48.0\",\n\t\t\t\t\t\"Remote\": false\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"Name\": \"truenas\",\n\t\t\t\t\t\"Version\": \"0.7.7\",\n\t\t\t\t\t\"Remote\": true\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\t\"Name\": \"zfs\",\n\t\t\t\t\t\"Version\": \"2.3.2-2\",\n\t\t\t\t\t\"Remote\": false\n\t\t\t\t}\n\t\t\t]\n\t\t}\n\t}"
time="2026-07-21T12:06:06+02:00" level=debug msg="Connected to the websocket: ws://unix.socket/1.0/events"
time="2026-07-21T12:06:06+02:00" level=debug msg="Sending request to Incus" etag= method=POST url="http://unix.socket/1.0/cluster/members/incus1.man-da.de/state"
time="2026-07-21T12:06:06+02:00" level=debug msg="\n\t{\n\t\t\"action\": \"evacuate\",\n\t\t\"mode\": \"auto\"\n\t}"
time="2026-07-21T12:06:06+02:00" level=debug msg="Got operation from Incus"
time="2026-07-21T12:06:06+02:00" level=debug msg="\n\t{\n\t\t\"id\": \"9b0058b3-aa54-4fac-9447-8b7cb09b397c\",\n\t\t\"class\": \"task\",\n\t\t\"description\": \"Evacuating cluster member\",\n\t\t\"created_at\": \"2026-07-21T12:06:06.156148836+02:00\",\n\t\t\"updated_at\": \"2026-07-21T12:06:06.156148836+02:00\",\n\t\t\"status\": \"Running\",\n\t\t\"status_code\": 103,\n\t\t\"resources\": {},\n\t\t\"metadata\": {},\n\t\t\"may_cancel\": false,\n\t\t\"err\": \"\",\n\t\t\"location\": \"incus1.man-da.de\"\n\t}"
time="2026-07-21T12:06:06+02:00" level=debug msg="Sending request to Incus" etag= method=GET url="http://unix.socket/1.0/operations/9b0058b3-aa54-4fac-9447-8b7cb09b397c"
time="2026-07-21T12:06:06+02:00" level=debug msg="Got response struct from Incus"
time="2026-07-21T12:06:06+02:00" level=debug msg="\n\t{\n\t\t\"id\": \"9b0058b3-aa54-4fac-9447-8b7cb09b397c\",\n\t\t\"class\": \"task\",\n\t\t\"description\": \"Evacuating cluster member\",\n\t\t\"created_at\": \"2026-07-21T12:06:06.156148836+02:00\",\n\t\t\"updated_at\": \"2026-07-21T12:06:06.156148836+02:00\",\n\t\t\"status\": \"Running\",\n\t\t\"status_code\": 103,\n\t\t\"resources\": {},\n\t\t\"metadata\": {},\n\t\t\"may_cancel\": false,\n\t\t\"err\": \"\",\n\t\t\"location\": \"incus1.man-da.de\"\n\t}"
Error: Failed to evacuate instances: Failed to migrate instance "static" in project "default": Failed deleting instance on source member: Error deleting storage volume: Failed to remove '/var/lib/incus/storage-pools/tank/virtual-machines/static': remove /var/lib/incus/storage-pools/tank/virtual-machines/static: directory not empty

The directory is indeed not empty, but the affected virtual machine is running on incus4:

root@incus1:~# find /var/lib/incus/storage-pools/tank/virtual-machines/static -maxdepth 2 -ls
  9964006      4 d--x------   4 incus    root         4096 Jul  3 11:26 /var/lib/incus/storage-pools/tank/virtual-machines/static
  9964089      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-110049176.png
  9964074      0 lrwxrwxrwx   1 root     root           19 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/qemu.nvram -> OVMF_VARS.4MB.ms.fd
  9964076      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-3408201783.png
  9964015      4 dr-x------   6 incus    root         4096 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config
  9964045      4 -rwx------   1 incus    root         1453 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/install.sh
  9964037      4 dr-x------   2 incus    root         4096 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/udev
  9964024      4 -r--------   1 incus    root          753 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/agent.crt
  9964032      4 dr-x------   2 incus    root         4096 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/systemd
  9964047      4 dr-x------   2 incus    root         4096 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/files
  9964053      4 -rw-r--r--   1 incus    root         1491 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/agent.conf
  9964056      4 -r--------   1 incus    root            2 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/agent-mounts.json
  9964046      0 lrwxrwxrwx   1 root     root           11 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/lxd-agent -> incus-agent
  9964021      4 -rwx------   1 incus    root          741 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/incus-agent
  9964023      4 -r--------   1 incus    root          721 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/server.crt
  9964027      4 -r--------   1 incus    root          288 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/agent.key
  9964052      4 dr-x------   2 incus    root         4096 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/config/nics
  9964110      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-2337042043.png
  9964114      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-3543600016.png
  9964010      4 drwxr-xr-x   2 root     root         4096 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/templates
  9964012      4 -rw-r--r--   1 root     root           21 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/templates/hostname.tpl
  9964013      4 -rw-r--r--   1 root     root          140 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/templates/hosts.tpl
  9964098      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-28979939.png
  9964099      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-2689291545.png
  9964080      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-1684829512.png
  9964059      4 -rw-r--r--   1 root     root          753 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/agent.crt
  9964007      4 -rw-r--r--   1 root     root          529 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/metadata.yaml
  9964113      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-535646406.png
  9964014      8 -r--------   1 root     root         5250 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/backup.yaml
  9964078      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-3254099108.png
  9964094      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-1178973180.png
  9964109      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-3074292887.png
  9964107      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-2723584077.png
  9964087      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-890594052.png
  9964063      4 -rw-r--r--   1 root     root          721 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/agent-client.crt
  9964091      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-1955795958.png
  9964067      4 -rw-------   1 root     root          288 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/agent-client.key
  9964117      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-3490401464.png
  9964062      4 -rw-------   1 root     root          288 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/agent.key
  9964095      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-131331358.png
  9964069    528 -rw-------   1 incus    root       540672 Jul  6 09:11 /var/lib/incus/storage-pools/tank/virtual-machines/static/OVMF_VARS.4MB.ms.fd
  9964090      0 -rw-------   1 incus    root            0 Nov 26  2025 /var/lib/incus/storage-pools/tank/virtual-machines/static/screenshot-1745399607.png

incus1 shows:

root@incus1:~# journalctl --no-pager -r -u incus | head -n 100
Jul 21 13:09:15 incus1.man-da.de incusd[2736573]: time="2026-07-21T13:09:15+02:00" level=error msg="Failed to synchronize images" err="Failed transferring image \"5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8\" from \"incus1.man-da.de:8443\": open /var/lib/incus/images/5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8: no such file or directory" fingerprint=5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8 project=default
Jul 21 12:13:54 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:13:54+02:00" level=warning msg="Cluster member isn't responding" name=incus2.man-da.de
Jul 21 12:13:54 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:13:54+02:00" level=warning msg="Aborting heartbeat round" err="context canceled" mode=normal
Jul 21 12:09:15 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:09:15+02:00" level=error msg="Failed to synchronize images" err="Failed transferring image \"5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8\" from \"incus1.man-da.de:8443\": open /var/lib/incus/images/5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8: no such file or directory" fingerprint=5d2252f068069ee81cc0ece671f3a2cac09d9ba04d09b6b393c6641a3005f2e8 project=default
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to remove snapshot file" err="remove /var/lib/incus/virtual-machines/static/incus_root_snap.qcow2: no such file or directory"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close snapshot file" err="close /var/lib/incus/virtual-machines/static/incus_root_snap.qcow2: file already closed"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to remove snapshot file" err="remove /var/lib/incus/virtual-machines/accounting/incus_root_snap.qcow2: no such file or directory"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close snapshot file" err="close /var/lib/incus/virtual-machines/accounting/incus_root_snap.qcow2: file already closed"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to unmount instance" err="In use"
Jul 21 12:06:06 incus1.man-da.de incusd[2736573]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to unmount instance" err="In use"
Jul 21 11:59:04 incus1.man-da.de incusd[2736573]: time="2026-07-21T11:59:04+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"

incus4 shows:

root@incus4:~# journalctl -r -u incus
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:41 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:41+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:39 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:39+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 12:06:33 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:33+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 12:06:06 incus4.man-da.de incusd[2531057]: time="2026-07-21T12:06:06+02:00" level=warning msg="Failed to close websocket writer" err="websocket: write closed"
Jul 21 11:59:04 incus4.man-da.de incusd[2531057]: time="2026-07-21T11:59:04+02:00" level=warning msg="Failed to close socket" err="bad file descriptor"

Expected behavior

Evacuation to succeed. And probably not having this files on the evacuated host. ;-)

Steps to reproduce

This has happend 2 times to different virtual machines on our host incus1. Last time on 2026-07-13 to the virtual machine rp4.

[stgraber]

Is that directory a mount point? If not, then you're looking at a weird scenario where something unmounted the config volume of the VM, effectively causing Incus to write stuff directly to your host's filesystem rather than to the mounted config volume.

[stgraber]

The Incus error is correct and we shouldn't automatically wipe things in such situations.

If it's a mountpoint, then Incus is failing to unmount it somehow, but I'd guess that it's not a mountpoint and that those things go instead written directly onto the host filesystem.

If you have a way to reproduce that situation, that'd be great as we could then sort it out, but the issue may have happened at any point following the VM startup, so could have been quite a while back.