Skip to content

Commit 1a74fc6

Browse files
committed
Initial (hacky) performance tests
Create initial performance tests measuring the time between hotplug API request and vCPUs being available to the guest. Note: These tests are NOT designed to be merged or used in CI, they are merely investigative tests for the latency of vCPU hotplugging Signed-off-by: James Curtis <[email protected]>
1 parent e6c2c93 commit 1a74fc6

File tree

9 files changed

+279
-4
lines changed

9 files changed

+279
-4
lines changed

src/vmm/src/devices/pseudo/boot_timer.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ const MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE: u8 = 123;
1010
/// Pseudo device to record the kernel boot time.
1111
#[derive(Debug)]
1212
pub struct BootTimer {
13-
start_ts: TimestampUs,
13+
pub start_ts: TimestampUs,
1414
}
1515

1616
impl BootTimer {

src/vmm/src/lib.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -688,6 +688,12 @@ impl Vmm {
688688
self.resume_vcpu_threads(start_idx.into())?;
689689

690690
self.acpi_device_manager.notify_cpu_container()?;
691+
if let Some(devices::BusDevice::BootTimer(timer)) =
692+
self.get_bus_device(DeviceType::BootTimer, "BootTimer")
693+
{
694+
let mut locked_timer = timer.lock().expect("Poisoned lock");
695+
locked_timer.start_ts = utils::time::TimestampUs::default();
696+
}
691697

692698
Ok(new_machine_config)
693699
}

tests/conftest.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,9 @@ def rootfs_fxt(request, record_property):
361361
guest_kernel_linux_5_10 = pytest.fixture(
362362
guest_kernel_fxt, params=kernel_params("vmlinux-5.10*")
363363
)
364+
guest_kernel_linux_acpi_only = pytest.fixture(
365+
guest_kernel_fxt, params=kernel_params("vmlinux-5.10.221")
366+
)
364367
# Use the unfiltered selector, since we don't officially support 6.1 yet.
365368
# TODO: switch to default selector once we add full 6.1 support.
366369
guest_kernel_linux_6_1 = pytest.fixture(
@@ -394,6 +397,15 @@ def uvm_plain_rw(microvm_factory, guest_kernel_linux_5_10, rootfs_rw):
394397
return microvm_factory.build(guest_kernel_linux_5_10, rootfs_rw)
395398

396399

400+
@pytest.fixture
401+
def uvm_hotplug(microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw):
402+
"""Create a VM with ACPI enabled kernels only.
403+
kernel: 5.10
404+
rootfs: Ubuntu 22.04
405+
"""
406+
return microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
407+
408+
397409
@pytest.fixture
398410
def uvm_nano(uvm_plain):
399411
"""A preconfigured uvm with 2vCPUs and 256MiB of memory

tests/host_tools/1-cpu-hotplug.rules

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SUBSYSTEM=="cpu", ACTION=="add", ATTR{online}!="1", ATTR{online}="1"

tests/host_tools/hotplug.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
while :; do
6+
[[ -d /sys/devices/system/cpu/cpu1 ]] && break
7+
done
8+
9+
readarray -t offline_cpus < <(lscpu -p=cpu --offline | sed '/^#/d')
10+
11+
for cpu_idx in ${offline_cpus[@]}; do
12+
echo 1 >/sys/devices/system/cpu/cpu$cpu_idx/online
13+
done
14+
15+
/home/hotplug_time.o

tests/host_tools/hotplug_time.c

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
// Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Init wrapper for boot timing. It points at /sbin/init.
5+
6+
#include <fcntl.h>
7+
#include <sys/mman.h>
8+
#include <sys/types.h>
9+
#include <unistd.h>
10+
11+
// Base address values are defined in arch/src/lib.rs as arch::MMIO_MEM_START.
12+
// Values are computed in arch/src/<arch>/mod.rs from the architecture layouts.
13+
// Position on the bus is defined by MMIO_LEN increments, where MMIO_LEN is
14+
// defined as 0x1000 in vmm/src/device_manager/mmio.rs.
15+
#ifdef __x86_64__
16+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0xd0000000
17+
#endif
18+
#ifdef __aarch64__
19+
#define MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE 0x40000000
20+
#endif
21+
22+
#define MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE 123
23+
24+
int main() {
25+
int fd = open("/dev/mem", (O_RDWR | O_SYNC | O_CLOEXEC));
26+
int mapped_size = getpagesize();
27+
28+
char *map_base = mmap(NULL, mapped_size, PROT_WRITE, MAP_SHARED, fd,
29+
MAGIC_MMIO_SIGNAL_GUEST_BOOT_COMPLETE);
30+
31+
*map_base = MAGIC_VALUE_SIGNAL_GUEST_BOOT_COMPLETE;
32+
msync(map_base, mapped_size, MS_ASYNC);
33+
}

tests/host_tools/hotplug_time.o

879 KB
Binary file not shown.

tests/integration_tests/functional/test_vcpu_hotplug.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,19 @@
33

44
"""Integration tests for hotplugging vCPUs"""
55

6+
import platform
67
import re
78
import time
89

910
import pytest
1011

11-
from framework import microvm
1212
from framework.defs import MAX_SUPPORTED_VCPUS
13-
from framework.microvm import Serial
1413
from framework.utils_cpuid import check_guest_cpuid_output
1514

1615

16+
@pytest.mark.skipif(
17+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
18+
)
1719
@pytest.mark.parametrize("vcpu_count", [1, MAX_SUPPORTED_VCPUS - 1])
1820
def test_hotplug_vcpus(uvm_plain, vcpu_count):
1921
"""Test hotplugging works as intended"""
@@ -40,6 +42,9 @@ def test_hotplug_vcpus(uvm_plain, vcpu_count):
4042
)
4143

4244

45+
@pytest.mark.skipif(
46+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
47+
)
4348
@pytest.mark.parametrize(
4449
"vcpu_count", [-1, 0, MAX_SUPPORTED_VCPUS, MAX_SUPPORTED_VCPUS + 1]
4550
)
@@ -63,7 +68,7 @@ def test_negative_hotplug_vcpus(uvm_plain, vcpu_count):
6368
with pytest.raises(
6469
RuntimeError,
6570
match=re.compile(
66-
f"An error occurred when deserializing the json body of a request: invalid value: integer `-\\d+`, expected u8+"
71+
"An error occurred when deserializing the json body of a request: invalid value: integer `-\\d+`, expected u8+"
6772
),
6873
):
6974
uvm_plain.api.hotplug.put(Vcpu={"add": vcpu_count})
@@ -75,6 +80,9 @@ def test_negative_hotplug_vcpus(uvm_plain, vcpu_count):
7580
uvm_plain.api.hotplug.put(Vcpu={"add": vcpu_count})
7681

7782

83+
@pytest.mark.skipif(
84+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
85+
)
7886
@pytest.mark.parametrize("vcpu_count", [1, MAX_SUPPORTED_VCPUS - 1])
7987
def test_online_hotplugged_vcpus(uvm_plain, vcpu_count):
8088
"""Test that hotplugged CPUs can be onlined"""
Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Testing hotplug performance"""
5+
6+
import platform
7+
import re
8+
import time
9+
from pathlib import Path
10+
11+
import pandas
12+
import pytest
13+
14+
from host_tools.cargo_build import gcc_compile
15+
16+
17+
@pytest.mark.parametrize(
18+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
19+
)
20+
def test_custom_udev_rule_latency(
21+
microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count
22+
):
23+
"""Test the latency for hotplugging and booting CPUs in the guest"""
24+
api_durations = []
25+
onlining_durations = []
26+
print(f"Vcpu count: {vcpu_count}")
27+
for i in range(5):
28+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
29+
uvm_hotplug.jailer.extra_args.update({"no-seccomp": None})
30+
uvm_hotplug.help.enable_console()
31+
uvm_hotplug.spawn()
32+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
33+
uvm_hotplug.add_net_iface()
34+
uvm_hotplug.start()
35+
uvm_hotplug.ssh.run("rm /usr/lib/udev/rules.d/40-vm-hotadd.rules")
36+
uvm_hotplug.ssh.scp_put(
37+
Path("./host_tools/1-cpu-hotplug.rules"),
38+
Path("/usr/lib/udev/rules.d/1-cpu-hotplug.rules"),
39+
)
40+
41+
time.sleep(0.25)
42+
43+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
44+
time.sleep(0.25)
45+
_, stdout, _ = uvm_hotplug.ssh.run("dmesg")
46+
47+
# Extract API call duration
48+
api_duration = (
49+
float(
50+
re.findall(
51+
r"Total previous API call duration: (\d+) us\.",
52+
uvm_hotplug.log_data,
53+
)[-1]
54+
)
55+
/ 1000
56+
)
57+
58+
# Extract onlining timings
59+
start = float(
60+
re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0]
61+
)
62+
end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1])
63+
elapsed_time = (end - start) * 1000
64+
print(f"Api call duration: {api_duration} ms")
65+
print(f"Onlining duration: {elapsed_time} ms")
66+
api_durations.append(api_duration)
67+
onlining_durations.append(elapsed_time)
68+
uvm_hotplug.kill()
69+
time.sleep(1)
70+
71+
avg_api_duration = sum(api_durations) / 5
72+
avg_onlining_duration = sum(onlining_durations) / 5
73+
print(f"Averages for {vcpu_count} hotplugged vcpus:")
74+
print(f"\tAverage API call duration: {avg_api_duration} ms")
75+
print(f"\tAverage onliing duration: {avg_onlining_duration} ms")
76+
77+
78+
@pytest.mark.parametrize(
79+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
80+
)
81+
def test_default_udev_rule_latency(
82+
microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count
83+
):
84+
"""Test the latency for hotplugging and booting CPUs in the guest"""
85+
api_durations = []
86+
onlining_durations = []
87+
print(f"Vcpu count: {vcpu_count}")
88+
for i in range(5):
89+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
90+
uvm_hotplug.jailer.extra_args.update({"no-seccomp": None})
91+
uvm_hotplug.help.enable_console()
92+
uvm_hotplug.spawn()
93+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
94+
uvm_hotplug.add_net_iface()
95+
uvm_hotplug.start()
96+
97+
time.sleep(0.25)
98+
99+
_, stdout, _ = uvm_hotplug.ssh.run("ls /usr/lib/udev/rules.d")
100+
default_rule = re.search(r"40-vm-hotadd\.rules", stdout)
101+
assert default_rule is not None
102+
103+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
104+
time.sleep(0.25)
105+
_, stdout, _ = uvm_hotplug.ssh.run("dmesg")
106+
107+
# Extract API call duration
108+
api_duration = (
109+
float(
110+
re.findall(
111+
r"Total previous API call duration: (\d+) us\.",
112+
uvm_hotplug.log_data,
113+
)[-1]
114+
)
115+
/ 1000
116+
)
117+
118+
# Extract onlining timings
119+
start = float(
120+
re.findall(r"\[\s+(\d+\.\d+)\] CPU1 has been hot-added\n", stdout)[0]
121+
)
122+
end = float(re.findall(r"\[\s+(\d+\.\d+)\] \w+", stdout)[-1])
123+
elapsed_time = (end - start) * 1000
124+
print(f"Api call duration: {api_duration} ms")
125+
print(f"Onlining duration: {elapsed_time} ms")
126+
api_durations.append(api_duration)
127+
onlining_durations.append(elapsed_time)
128+
uvm_hotplug.kill()
129+
time.sleep(1)
130+
131+
avg_api_duration = sum(api_durations) / 5
132+
avg_onlining_duration = sum(onlining_durations) / 5
133+
print(f"Averages for {vcpu_count} hotplugged vcpus:")
134+
print(f"\tAverage API call duration: {avg_api_duration} ms")
135+
print(f"\tAverage onliing duration: {avg_onlining_duration} ms")
136+
137+
138+
@pytest.mark.skipif(
139+
platform.machine() != "x86_64", reason="Hotplug only enabled on x86_64."
140+
)
141+
@pytest.mark.parametrize(
142+
"vcpu_count", [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30]
143+
)
144+
def test_manual_latency(
145+
microvm_factory, guest_kernel_linux_acpi_only, rootfs_rw, vcpu_count
146+
):
147+
"""Test the latency for hotplugging and booting CPUs in the guest"""
148+
gcc_compile(Path("./host_tools/hotplug_time.c"), Path("host_tools/hotplug_time.o"))
149+
data = []
150+
for _ in range(50):
151+
uvm_hotplug = microvm_factory.build(guest_kernel_linux_acpi_only, rootfs_rw)
152+
uvm_hotplug.jailer.extra_args.update({"boot-timer": None, "no-seccomp": None})
153+
uvm_hotplug.help.enable_console()
154+
uvm_hotplug.spawn()
155+
uvm_hotplug.basic_config(vcpu_count=1, mem_size_mib=128)
156+
uvm_hotplug.add_net_iface()
157+
uvm_hotplug.start()
158+
uvm_hotplug.ssh.scp_put(
159+
Path("./host_tools/hotplug.sh"), Path("/home/hotplug.sh")
160+
)
161+
uvm_hotplug.ssh.scp_put(
162+
Path("./host_tools//hotplug_time.o"), Path("/home/hotplug_time.o")
163+
)
164+
uvm_hotplug.ssh.run(
165+
"tmux new-session -d /bin/bash /home/hotplug.sh > /home/test 2>&1"
166+
)
167+
168+
uvm_hotplug.api.hotplug.put(Vcpu={"add": vcpu_count})
169+
170+
time.sleep(1.5)
171+
# Extract API call duration
172+
api_duration = (
173+
float(
174+
re.findall(
175+
r"Total previous API call duration: (\d+) us\.",
176+
uvm_hotplug.log_data,
177+
)[-1]
178+
)
179+
/ 1000
180+
)
181+
try:
182+
timestamp = (
183+
float(
184+
re.findall(
185+
r"Guest-boot-time\s+\=\s+(\d+)\s+us", uvm_hotplug.log_data
186+
)[0]
187+
)
188+
/ 1000
189+
)
190+
except IndexError:
191+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": None})
192+
continue
193+
# Extract onlining timings
194+
data.append({"vcpus": vcpu_count, "api": api_duration, "onlining": timestamp})
195+
196+
df = pandas.DataFrame.from_dict(data).to_csv(
197+
f"../test_results/manual-hotplug_{vcpu_count}.csv",
198+
index=False,
199+
float_format="%.3f",
200+
)

0 commit comments

Comments
 (0)