Skip to content

qemu: support nestedVirtualization with HVF (QEMU 11.1 or later) #5443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sharanrajt wants to merge 1 commit into
base: master
Choose a base branch
from
+58 −1
Open

qemu: support nestedVirtualization with HVF (QEMU 11.1 or later) #5443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions pkg/driver/qemu/qemu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,15 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
return "", nil, fmt.Errorf("accelerator %#q is not supported by %s", accel, exe)
}

if y.NestedVirtualization != nil && *y.NestedVirtualization && accel == "hvf" {
if version != nil {
minNestedVirtVersion := semver.New("11.1.0")
if version.LessThan(*minNestedVirtVersion) {
logrus.Fatalf("QEMU %v is too old, %v or later is required for nested virtualization with HVF", version, minNestedVirtVersion)
}
}
}

// Memory
memBytes, err := units.RAMInBytes(*y.Memory)
if err != nil {
Expand Down Expand Up @@ -550,6 +559,15 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
}
args = appendArgsIfNoConflict(args, "-cpu", cpu)

// Nested virtualization is only supported for the `virt` machine of aarch64 and armv7l (`virtualization=on`).
if y.NestedVirtualization != nil && *y.NestedVirtualization {
switch *y.Arch {
case limatype.AARCH64, limatype.ARMV7L:
default:
logrus.Warnf("field `nestedVirtualization` is not supported for architecture %#q, ignoring", *y.Arch)
}
}

// Machine
switch *y.Arch {
case limatype.X8664:
Expand All @@ -570,6 +588,9 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
}
case limatype.AARCH64:
machine := "virt,accel=" + accel
if y.NestedVirtualization != nil && *y.NestedVirtualization {
machine += ",virtualization=on"
}
args = appendArgsIfNoConflict(args, "-machine", machine)
case limatype.RISCV64:
// https://github.com/tianocore/edk2/blob/edk2-stable202408/OvmfPkg/RiscVVirt/README.md#test
Expand All @@ -581,6 +602,9 @@ func Cmdline(ctx context.Context, cfg Config) (exe string, args []string, err er
args = appendArgsIfNoConflict(args, "-machine", machine)
case limatype.ARMV7L:
machine := "virt,accel=" + accel
if y.NestedVirtualization != nil && *y.NestedVirtualization {
machine += ",virtualization=on"

@AkihiroSuda AkihiroSuda Aug 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How did you test this configuration?

@sharanrajt sharanrajt Aug 27, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't test it. I added it because the virt machine accepts virtualization=on for 32-bit too. But armv7l guests aretcg-only in Lima and Linux dropped kvm on 32-bit ARM hosts in 5.7, so a guest couldn't use it anyway. I'll remove the armv7l case and keep this PR to aarch64.

}
args = appendArgsIfNoConflict(args, "-machine", machine)
case limatype.PPC64LE:
machine := "pseries,accel=" + accel
Expand Down
12 changes: 12 additions & 0 deletions pkg/driver/qemu/qemu_driver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,18 @@ func validateConfig(cfg *limatype.LimaYAML) error {
}
}

if cfg.NestedVirtualization != nil && *cfg.NestedVirtualization {

@AkihiroSuda AkihiroSuda Aug 26, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we support Linux hosts?

@sharanrajt sharanrajt Aug 27, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. For aarch64 the same code path applies on Linux/KVM, the macOS version check is darwin-only, and virtualization=on works with KVM given QEMU ≥ 10.1 and a ≥ 6.16 host kernel booted with kvm-arm.mode=nested (FEAT_NV hardware); otherwise QEMU fails with a clear "host kernel KVM does not support providing Virtualization extensions" error. I don't have such hardware, so this part is untested.
For x86_64/KVM, nested virt is a host-side setting exposed through -cpu host, which Lima already uses. I'll change the x86_64 path to check /sys/module/kvm_{intel,amd}/parameters/nested and error only when it's disabled, instead of warning "unsupported". I'll also document the Linux requirements in default.yaml.

if runtime.GOOS == "darwin" {
macOSProductVersion, err := osutil.ProductVersion()
if err != nil {
return err
}
if macOSProductVersion.LessThan(*semver.New("15.0.0")) {
return errors.New("nested virtualization requires macOS 15 or newer")
}
}
}

return nil
}

Expand Down
21 changes: 21 additions & 0 deletions pkg/driver/qemu/qemu_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@ import (
"runtime"
"testing"

"github.com/coreos/go-semver/semver"
"gotest.tools/v3/assert"

"github.com/lima-vm/lima/v2/pkg/limatype"
"github.com/lima-vm/lima/v2/pkg/limatype/filenames"
"github.com/lima-vm/lima/v2/pkg/osutil"
"github.com/lima-vm/lima/v2/pkg/ptr"
)

func TestArgValue(t *testing.T) {
Expand Down Expand Up @@ -151,3 +154,21 @@ func TestSwtpmCmdline(t *testing.T) {
_, err = os.Stat(swtpmSock)
assert.ErrorIs(t, err, os.ErrNotExist)
}

func TestValidateConfigNestedVirtualization(t *testing.T) {
cfg := &limatype.LimaYAML{
NestedVirtualization: ptr.Of(true),
}
err := validateConfig(cfg)
if runtime.GOOS == "darwin" {
productVer, verErr := osutil.ProductVersion()
assert.NilError(t, verErr)
if productVer.LessThan(*semver.New("15.0.0")) {
assert.ErrorContains(t, err, "nested virtualization requires macOS 15 or newer")
} else {
assert.NilError(t, err)
}
} else {
assert.NilError(t, err)
}
}
2 changes: 1 addition & 1 deletion templates/default.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -657,7 +657,7 @@ plain: null
# qemu-system-aarch64 -accel kvm -cpu host -M virt
# - Without specifying `-cpu host`, nested virtualization may fail with the error:
# qemu-system-aarch64: kvm_init_vcpu: kvm_arch_init_vcpu failed (0): Invalid argument
# - Only supported on Apple M3 or later with `vmType: vz` or `vmType: krunkit`.
# - Only supported on Apple M3 or later.
# 🟢 Builtin default: false
nestedVirtualization: null

Expand Down
Loading