Skip to content

Commit 7c1ab47

Browse files
kkourtkkourt
committed
scripts for kernel dev with KVM
Signed-off-by: Kornilios Kourtis <[email protected]>
0 parents  commit 7c1ab47

File tree

8 files changed

+368
-0
lines changed

8 files changed

+368
-0
lines changed

‎README.md

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
2+
First, we can edit the `conf` file for the various env variables.
3+
4+
# Images
5+
6+
Next, we need to build the root fs:
7+
8+
```
9+
$ ./scripts/mkroot.sh
10+
```
11+
12+
13+
After that, we install `kubeadm`:
14+
```
15+
$ ./scripts/mkkubeadm.sh
16+
```
17+
18+
Next, build the images for the two vms:
19+
```
20+
./scripts/mkkvm.sh 0
21+
./scripts/mkkvm.sh 1
22+
```
23+
24+
# Networking
25+
26+
(Proper scripts to be added)
27+
28+
There are multiple ways to setup networking. I opted for using a bridge device,
29+
and attach a NIC for all VMs there. (to be automated)
30+
31+
```
32+
# sudo ip link add kvm-br type bridge
33+
# sudo ip addr add 10.33.33.100/24 dev kvm-br
34+
```
35+
36+
And then, masquarade traffic from the VMs to the outside world
37+
38+
```
39+
# sudo sh -c 'echo 0 > /proc/sys/net/bridge/bridge-nf-call-iptables'
40+
# sudo modprobe br_netfilter
41+
# sudo sysctl -w net.ipv4.ip_forward=1
42+
# sudo iptables -t nat -A POSTROUTING -o enxa0cec8e6e64c -j MASQUERADE
43+
# sudo iptables -A FORWARD -i enxa0cec8e6e64c -o kvm-br -m state --state RELATED,ESTABLISHED -j ACCEPT
44+
# sudo iptables -A FORWARD -i kvm-br -o enxa0cec8e6e64c -j ACCEPT
45+
```
46+
47+
I use a simple `dnsmasq.conf`:
48+
49+
```
50+
interface=kvm-br
51+
port=0 # no dns
52+
53+
dhcp-option=option:router,10.33.33.100
54+
dhcp-option=option:dns-server,8.8.8.8
55+
56+
dhcp-range=10.33.33.0,static
57+
dhcp-host=52:54:00:12:34:01,kvm1,10.33.33.1
58+
dhcp-host=52:54:00:12:34:02,kvm2,10.33.33.2
59+
60+
```
61+
62+
That I run in debug mode:
63+
```
64+
sudo dnsmasq --conf-file=dnsmasq.conf -d
65+
```
66+
67+
# Kernel image
68+
69+
The kernel image is expected to be in `KERNEL_SOURCE` in `conf`. Run
70+
`./scripts/installmods.sh` to install modules.
71+
72+
73+
# start vms
74+
75+
```
76+
./scripts/runkvm 0
77+
```
78+
79+
```
80+
./scripts/runkvm 1
81+
```
82+
83+
84+
# Notes
85+
86+
`0` and `1` in `runkvm` and `mkkvm` scripts are used as indices to respective
87+
bash arrays.

‎conf

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
NBDDEV="/dev/nbd0"
3+
4+
ROOTIMG="./images/rootfs.qcow2"
5+
ROOTDIR="./rootfs"
6+
PACKAGES=("less" "vim" "sudo" "openssh-server" "python3" "docker.io" "curl")
7+
8+
KUBEADMIMG="./images/kubeadmfs.qcow2"
9+
KUBEADMDIR="./kubeadmfs"
10+
11+
BRIDGE_IFACE="kvm-br"
12+
VM_IMAGES=("./images/kvm1.qcow2" "./images/kvm2.qcow2")
13+
VM_NAMES=("kvm1" "kvm2")
14+
VM_DIRS=("kvm1fs" "kvm2fs")
15+
VM_NICS_MACS=("52:54:00:12:34:01" "52:54:00:12:34:02")
16+
17+
KERNEL_SOURCE=/usr/src/linux.kvm

‎dnsmasq.conf

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# only listen to the specified interface to avoid conflicts
2+
interface=kvm-br
3+
bind-interfaces
4+
#port=5353 # custom port not needed
5+
no-daemon
6+
7+
dhcp-option=option:router,10.33.33.100
8+
dhcp-option=option:dns-server,10.33.33.100
9+
10+
dhcp-range=10.33.33.0,static
11+
dhcp-host=52:54:00:12:34:01,kvm1,10.33.33.1,infinite
12+
dhcp-host=52:54:00:12:34:02,kvm2,10.33.33.2,infinite
13+
14+
address=/kvm1/10.33.33.1
15+
address=/kvm2/10.33.33.2

‎scripts/installmods.sh

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
# vim:noexpandtab
3+
4+
set -eu -o pipefail
5+
set -x
6+
7+
CONF_DIR="$(realpath $(dirname "${BASH_SOURCE[0]}")/..)"
8+
source $CONF_DIR/conf
9+
10+
sudo make -C $KERNEL_SOURCE INSTALL_MOD_PATH=$CONF_DIR/modules modules_install

‎scripts/mkkubeadm.sh

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
#!/bin/bash
2+
# vim:noexpandtab
3+
4+
set -eu -o pipefail
5+
set -x
6+
7+
CONF_DIR="$(realpath $(dirname "${BASH_SOURCE[0]}")/..)"
8+
source $CONF_DIR/conf
9+
10+
mkkubeadm() {
11+
if [ -f "$KUBEADMIMG" ]; then
12+
set +x
13+
echo "file $KUBEADMIMG already exists. Bailing out"
14+
exit 1
15+
fi
16+
17+
sudo modprobe nbd
18+
mkdir -p $KUBEADMDIR
19+
sudo qemu-img create -f qcow2 -b $(basename $ROOTIMG) $KUBEADMIMG
20+
sudo qemu-nbd -d $NBDDEV
21+
sudo qemu-nbd -c $NBDDEV $KUBEADMIMG
22+
sudo mount $NBDDEV $KUBEADMDIR
23+
}
24+
25+
xinstall() {
26+
sudo chroot $KUBEADMDIR sh <<- ENDCHROOT
27+
set -e
28+
set -x
29+
30+
cat <<-EOF | sudo tee /etc/sysctl.d/k8s.conf
31+
net.bridge.bridge-nf-call-ip6tables = 1
32+
net.bridge.bridge-nf-call-iptables = 1
33+
net.ipv4.ip_forward = 1
34+
EOF
35+
sudo sysctl --system
36+
37+
sudo apt-get update && sudo apt-get install -y apt-transport-https curl gnupg
38+
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
39+
cat <<-EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
40+
deb https://apt.kubernetes.io/ kubernetes-xenial main
41+
EOF
42+
sudo apt-get update
43+
44+
sudo apt-get install -y docker.io
45+
46+
sudo apt-get install -y kubelet kubeadm kubectl
47+
sudo apt-mark hold kubelet kubeadm kubectl
48+
49+
#sudo systemctl enable docker.service
50+
#sudo systemctl daemon-reload
51+
#sudo systemctl restart kubelet
52+
ENDCHROOT
53+
}
54+
55+
cleanup() {
56+
sudo umount $KUBEADMDIR
57+
sudo qemu-nbd -d $NBDDEV
58+
}
59+
60+
if [ -f $KUBEADMIMG ]; then
61+
echo "file $KUBEADMIMG already exists: bailing out"
62+
exit 1
63+
fi
64+
65+
mkkubeadm
66+
xinstall
67+
cleanup

‎scripts/mkkvm.sh

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
# vim:noexpandtab
3+
4+
set -eu -o pipefail
5+
set -x
6+
7+
CONF_DIR="$(realpath $(dirname "${BASH_SOURCE[0]}")/..)"
8+
source $CONF_DIR/conf
9+
10+
mkkvm() {
11+
local kvmimage=${VM_IMAGES[$1]}
12+
local kvmdir=${VM_DIRS[$1]}
13+
14+
if [ -f "$kvmimage" ]; then
15+
set +x
16+
echo "file $kvmimage already exists. Bailing out"
17+
exit 1
18+
fi
19+
20+
sudo modprobe nbd
21+
mkdir -p $kvmdir
22+
#sudo qemu-img create -f qcow2 -b $(basename $KUBEADMIMG) $kvmimage
23+
cp $KUBEADMIMG $kvmimage
24+
sudo qemu-nbd -d $NBDDEV
25+
sudo qemu-nbd -c $NBDDEV $kvmimage
26+
sudo mount $NBDDEV $kvmdir
27+
}
28+
29+
cleanup() {
30+
local kvmdir=${VM_DIRS[$1]}
31+
sync
32+
sudo umount $kvmdir
33+
sudo qemu-nbd -d $NBDDEV
34+
}
35+
36+
xconfig() {
37+
local xname=${VM_NAMES[$1]}
38+
local kvmdir=${VM_DIRS[$1]}
39+
sudo sh -c "echo $xname > $kvmdir/etc/hostname"
40+
}
41+
42+
main() {
43+
mkkvm $1
44+
xconfig $1
45+
cleanup $1
46+
}
47+
48+
49+
main $1

‎scripts/mkroot.sh

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
# vim:noexpandtab
3+
4+
set -eu -o pipefail
5+
set -x
6+
7+
CONF_DIR="$(realpath $(dirname "${BASH_SOURCE[0]}")/..)"
8+
source $CONF_DIR/conf
9+
10+
function mkimage() {
11+
if [ -f "$ROOTIMG" ]; then
12+
set +x
13+
echo "file $ROOTIMG already exists. Bailing out"
14+
exit 1
15+
fi
16+
17+
sudo modprobe nbd
18+
mkdir -p $ROOTDIR
19+
mkdir -p $(dirname $ROOTIMG)
20+
sudo qemu-img create -f qcow2 $ROOTIMG 8G
21+
sudo qemu-nbd -d $NBDDEV
22+
sudo qemu-nbd -c $NBDDEV $ROOTIMG
23+
sudo mkfs.ext4 $NBDDEV
24+
sudo mount $NBDDEV $ROOTDIR
25+
sudo debootstrap --include=$(IFS=, ; echo "${PACKAGES[*]}") sid $ROOTDIR
26+
}
27+
28+
function mntimage() {
29+
sudo modprobe nbd
30+
mkdir -p $ROOTDIR
31+
sudo qemu-nbd -d $NBDDEV
32+
sudo qemu-nbd -c $NBDDEV $ROOTIMG
33+
sudo mount $NBDDEV $ROOTDIR
34+
}
35+
36+
function chrootconfig() {
37+
sudo chroot $ROOTDIR sh <<- ENDCHROOT
38+
apt-get update
39+
apt-get upgrade
40+
passwd -d root
41+
update-alternatives --set iptables /usr/sbin/iptables-legacy
42+
ENDCHROOT
43+
}
44+
45+
function miscconfig() {
46+
47+
cat <<- ENDSSHDCONFIG | sudo tee -a $ROOTDIR/etc/ssh/sshd_config
48+
PermitRootLogin yes
49+
ENDSSHDCONFIG
50+
51+
sudo mkdir $ROOTDIR/root/.ssh
52+
cat ~/.ssh/id_rsa.pub | sudo tee -a $ROOTDIR/root/.ssh/authorized_keys
53+
54+
cat <<- ENDBASHRC | sudo tee -a $ROOTDIR/root/.bashrc
55+
alias k='kubectl'
56+
alias ks='kubectl -n kube-system'
57+
alias kslogs='kubectl -n kube-system logs -l k8s-app=cilium --tail=-1'
58+
cilium_pod() {
59+
kubectl -n kube-system get pods -l k8s-app=cilium -o jsonpath="{.items[?(@.spec.nodeName == \"\$1\")].metadata.name}"
60+
}
61+
ENDBASHRC
62+
63+
cat <<- ENDFSTAB | sudo tee -a $ROOTDIR/etc/fstab
64+
modules /lib/modules 9p trans=virtio,ro 0 0
65+
ENDFSTAB
66+
67+
cat <<- ENDIFACES | sudo tee -a $ROOTDIR/etc/network/interfaces
68+
auto enp0s4
69+
iface enp0s4 inet dhcp
70+
auto enp0s3
71+
iface enp0s3 inet dhcp
72+
ENDIFACES
73+
}
74+
75+
cleanup() {
76+
sudo umount $ROOTDIR
77+
sudo qemu-nbd -d $NBDDEV
78+
}
79+
80+
if [ -f $ROOTIMG ]; then
81+
echo "file $ROOTIMG already exists: bailing out"
82+
exit 1
83+
fi
84+
85+
mkimage
86+
chrootconfig
87+
miscconfig
88+
cleanup

‎scripts/runkvm.sh

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#!/bin/bash
2+
# vim:noexpandtab
3+
4+
set -eu -o pipefail
5+
set -x
6+
7+
CONF_DIR="$(realpath $(dirname "${BASH_SOURCE[0]}")/..)"
8+
source $CONF_DIR/conf
9+
10+
kvm() {
11+
local image=${VM_IMAGES[$1]}
12+
local netdev_mac=${VM_NICS_MACS[$1]}
13+
local netdev="e1000"
14+
#local netdev="virtio-net"
15+
16+
declare -a qemu_opts=("-enable-kvm" "-m" "2G" "-smp" "2")
17+
qemu_opts+=("-kernel" "$KERNEL_SOURCE/arch/x86/boot/bzImage")
18+
qemu_opts+=("-append" "root=/dev/vda rw console=hvc0")
19+
qemu_opts+=("-drive" "file=$image,media=disk,if=virtio")
20+
qemu_opts+=("-chardev" "stdio,id=stdio,mux=on,signal=off")
21+
qemu_opts+=("-device" "virtio-serial-pci")
22+
qemu_opts+=("-device" "virtconsole,chardev=stdio")
23+
qemu_opts+=("-mon" "chardev=stdio")
24+
qemu_opts+=("-display" "none")
25+
qemu_opts+=("-netdev" "bridge,br=${BRIDGE_IFACE},id=n0" "-device" "$netdev,netdev=n0,mac=$netdev_mac")
26+
##qemu_opts+=("-net user,hostfwd=tcp:127.0.0.1:5001-:22")
27+
qemu_opts+=("-fsdev" "local,id=fs1,path=/home,security_model=none")
28+
qemu_opts+=("-device" "virtio-9p-pci,fsdev=fs1,mount_tag=home")
29+
qemu_opts+=("-fsdev" "local,id=fs2,path=$CONF_DIR/modules/lib/modules,security_model=none")
30+
qemu_opts+=("-device" "virtio-9p-pci,fsdev=fs2,mount_tag=modules")
31+
32+
sudo qemu-system-x86_64 "${qemu_opts[@]}"
33+
}
34+
35+
kvm $1

0 commit comments

Comments
 (0)