Merge pull request #2 from TUM-DSE/dev/measure3

- automate multi-host measurements
- add Interface.VPP
- add filter VNF to measurements
- restructure unikraft builds a bit more

Author: pogobanane

.config.click_qemu-x86_64.known-working

@@ -2,7 +2,6 @@
 build/
 nix/builds/
 target
-.config
 .unikraft
 .config.click_qemu-x86_64
 .config.click_qemu-x86_64.old
@@ -18,7 +17,7 @@ libs/libcxxabi/
 libs/libunwind/
 libs/musl/
 libs/openssl/
-libs/unikraft/
+libs/click-og/
 
 VMs/
 out*/

.gitignore

@@ -4,3 +4,6 @@
 [submodule "baseline/click"]
 	path = baseline/click
 	url = [email protected]:kohler/click.git
+[submodule "libs/unikraft"]
+	path = libs/unikraft
+	url = https://github.com/unikraft/unikraft.git

.gitmodules

@@ -4,30 +4,4 @@ build:
 	mkdir -p .unikraft/build/libclick/origin/click-a5384835a6cac10f8d44da4eeea8eaa8f8e6a0c2/elements/unikraft || true
 	cp -r libs/click/unikraft .unikraft/build/libclick/origin/click-a5384835a6cac10f8d44da4eeea8eaa8f8e6a0c2/elements
 
-	kraft build --log-type basic
-downloadLibs:
-	@nix develop .#unikraft --command bash -c 'sourceRoot=$$(pwd); eval "$$postUnpack"'
-
-kill:
-		sudo pkill -f "clicknet"
-		sudo pkill -f "controlnet"
-
-throughput.cpio: throughput.click
-	rm -r /tmp/ukcpio-$(USER) || true
-	mkdir -p /tmp/ukcpio-$(USER)
-	cp ./throughput.click /tmp/ukcpio-$(USER)/config.click
-	./libs/unikraft/support/scripts/mkcpio ./throughput.cpio /tmp/ukcpio-$(USER)
-
-vm: throughput.cpio
-	sudo taskset -c 3,4 qemu-system-x86_64 -accel kvm -cpu max -netdev bridge,id=en0,br=clicknet -device virtio-net-pci,netdev=en0 -append " vfs.fstab=[\"initrd0:/:extract::ramfs=1:\"] --" -kernel ./.unikraft/build/click_qemu-x86_64 -initrd ./throughput.cpio -nographic
-
-perf-kvm-top:
-	sudo perf kvm --guestkallsyms=./.unikraft/build/click_qemu-x86_64.sym --guestvmlinux=.unikraft/build/click_qemu-x86_64.dbg top -p $(pgrep qemu)
-
-perf-kvm-record:
-	sudo perf kvm --guestkallsyms=./.unikraft/build/click_qemu-x86_64.sym --guestvmlinux=.unikraft/build/click_qemu-x86_64.dbg record -g -p $(pgrep qemu)
-	sudo perf kvm --guestkallsyms=./.unikraft/build/click_qemu-x86_64.sym --guestvmlinux=.unikraft/build/click_qemu-x86_64.dbg report
-
-perf-qemu-record:
-	sudo perf record -g -p $(pgrep qemu)
-	sudo perf script > perf.trace
+	kraft build --log-type basic $(EXTRA_KRAFT_ARGS)

Makefile

@@ -29,7 +29,7 @@ The project is structured as follows:
 1. Clone the repository with the `--recursive` flag to also clone the submodules
 2. Install the `nix` package manager
 3. Run `nix develop .#fhs` to enter the development environment with all system dependencies
-4. Run `make downloadLibs` to download unikraft libraries as pinned by `flake.*`
+4. Run `just downloadLibs` to download unikraft libraries as pinned by `flake.*`
 
 ### Building the Unikernel
 
@@ -72,7 +72,11 @@ make sync
 You can run the benchmarks:
 
 ```
-python3 benchmark/pysrc/measure_throughput.py -c benchmark/conf/autotest_localhost.cfg -vvv
+# localhost doesn't support Interface.VPP (vhost-user):
+python3 benchmark/pysrc/measure_throughput.py -c benchmark/conf/uk_localhost.cfg -vvv
+
+# multihost:
+python3 benchmark/pysrc/measure_throughput.py -c benchmark/conf/uk_adelaide_wilfred.cfg -vvv
 ```
 
 See `-h` for more benchmarking options. When run on other hosts than `adelaide`, the config file and `benchmark/conf/ssh_config_doctor_cluster` may need adjusting.

README.md

@@ -1,91 +1,71 @@
-Host vm.local
-Hostname 192.168.56.20
-User root
+Host *
 StrictHostKeyChecking=no
-IdentityFile /home/okelmann/appclick-ubpf/nix/ssh_key
 UserKnownHostsFile=/dev/null
 IdentityAgent=/dev/null
 StrictHostKeyChecking=no
 LogLevel=ERROR
 
-Host guest.vmux.local
-Hostname 192.168.56.20
-User root
-StrictHostKeyChecking=no
-IdentityFile /scratch/okelmann/vmuxIO/nix/ssh_key
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-StrictHostKeyChecking=no
-LogLevel=ERROR
+
+# hosts in the doctor-cluster that are accessible if python is run on christina
+
+Host christina.christina
+Hostname christina.dos.cit.tum.de
+IdentityFile /root/.ssh/lawful_access
+
+Host adelaide.christina
+Hostname adelaide.dos.cit.tum.de
+IdentityFile /root/.ssh/lawful_access
+
+Host rose.christina
+Hostname rose.dos.cit.tum.de
+IdentityFile /root/.ssh/lawful_access
 
 Host rose.christina
 Hostname rose.dos.cit.tum.de
 IdentityFile /root/.ssh/lawful_access
-StrictHostKeyChecking=no
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-StrictHostKeyChecking=no
-LogLevel=ERROR
 
 Host wilfred.christina
 Hostname wilfred.dos.cit.tum.de
 IdentityFile /root/.ssh/lawful_access
-StrictHostKeyChecking=no
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-StrictHostKeyChecking=no
-LogLevel=ERROR
 
 Host river.christina
 Hostname river.dos.cit.tum.de
 IdentityFile /root/.ssh/lawful_access
-StrictHostKeyChecking=no
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
 
 Host amy.christina
 Hostname amy.dos.cit.tum.de
 IdentityFile /root/.ssh/lawful_access
-StrictHostKeyChecking=no
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
+
+
+# VMs
+
+Host vm*
+IdentityFile /home/okelmann/appclick-ubpf/nix/ssh_key
+
+Host vm.local
+Hostname 192.168.56.20
+User root
+
+Host guest.vmux.local
+Hostname 192.168.56.20
+User root
+IdentityFile /home/okelmann/appclick-ubpf/nix/ssh_key
+
+Host vm*.adelaide.christina
+ProxyJump adelaide.christina
+User root
 
 Host vm*.rose.christina
 ProxyJump rose.christina
 User root
-StrictHostKeyChecking=no
-IdentityFile /scratch/okelmann/vmuxIO/nix/ssh_key
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
 
 Host vm*.amy.christina
 ProxyJump amy.christina
 User root
-StrictHostKeyChecking=no
-IdentityFile /scratch/okelmann/vmuxIO/nix/ssh_key
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
 
 Host vm.river.christina
 ProxyJump river.christina
 User root
-StrictHostKeyChecking=no
-IdentityFile /scratch/okelmann/vmuxIO/nix/ssh_key
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
-
-Host christina.christina
-Hostname christina.dos.cit.tum.de
-IdentityFile /root/.ssh/lawful_access
-StrictHostKeyChecking=no
-UserKnownHostsFile=/dev/null
-IdentityAgent=/dev/null
-LogLevel=ERROR
 
 
 # generate vm*.* hosts with:

benchmark/conf/ssh_config_doctor_cluster

@@ -0,0 +1,63 @@
+[common]
+tmux_socket=vmux-${username}.tmux.sock
+
+[local]
+moonprogs_dir = ${projectDirectory}/test/moonprogs/
+
+[host]
+fqdn=${username}@adelaide.christina
+admin_bridge=bra-${username}0
+admin_bridge_ip_net=192.168.56.1/21
+admin_tap=tapa-${username}0
+test_iface=eno1
+test_iface_addr=0000:01:00.0
+test_iface_mac=3c:ec:ef:62:ad:f2
+test_iface_driv=ixgbe
+test_iface_dpdk_driv=vfio-pci
+test_iface_vfio_driv=vfio-pci
+test_bridge=br-${username}
+test_bridge_ip_net=10.1.0.1/21
+test_tap=tap-${username}
+test_macvtap=mvtap-${username}1
+vmux_socket_path=/tmp/vpp-${username}.sock
+tmux_socket=${common:tmux_socket}
+moongen_dir=${projectDirectory}/mgln/
+project_root=${projectDirectory}
+xdp_reflector_dir=${projectDirectory}/xdp/lib/
+qemu_path=${projectDirectory}/nix/builds/qemu/bin
+vmux_qemu_path=
+root_disk_file=${projectDirectory}/VMs/guest-image.qcow2
+fsdevs=home:${projectDirectoryParent},nixstore:/nix/store
+ssh_config=${projectDirectory}/benchmark/conf/ssh_config_doctor_cluster
+ssh_as_root=True
+vmux_path=${projectDirectory}/build_release/vmux
+
+[guest]
+fqdn=vm.adelaide.christina
+vcpus=1
+memory=8192
+admin_iface_mac=52:54:00:ea:00:0f
+test_iface=ens7
+test_iface_addr=0000:00:07.0
+test_iface_mac=52:54:00:fa:00:60
+test_iface_ip_net=10.1.0.20/21
+test_iface_driv=virtio-pci
+test_iface_dpdk_driv=igb_uio
+tmux_socket=${common:tmux_socket}
+moongen_dir=/root/${projectDirectoryName}/mgln/
+project_root=/root/${projectDirectoryName}
+xdp_reflector_dir=/root/${projectDirectoryName}/xdp/lib/
+ssh_config=/root/${projectDirectoryName}/benchmark/conf/ssh_config_doctor_cluster
+
+[loadgen] # is christina
+fqdn=localhost
+test_iface=eno2
+test_iface_addr=0000:01:00.1
+test_iface_mac=3c:ec:ef:62:ad:7d
+test_iface_ip_net=10.1.0.2/21
+test_iface_driv=ixgbe
+test_iface_dpdk_driv=vfio-pci
+tmux_socket=${common:tmux_socket}
+moongen_dir=${projectDirectory}/mgln/
+project_root=${projectDirectory}
+ssh_config=${projectDirectory}/benchmark/conf/ssh_config_doctor_cluster

benchmark/conf/uk_adelaide_christina.cfg

@@ -24,7 +24,7 @@ tmux_socket=${common:tmux_socket}
 moongen_dir=${projectDirectory}/mgln/
 project_root=${projectDirectory}
 xdp_reflector_dir=${projectDirectory}/xdp/lib/
-qemu_path=
+qemu_path=${projectDirectory}/nix/builds/qemu/bin
 vmux_qemu_path=
 root_disk_file=${projectDirectory}/VMs/guest-image.qcow2
 fsdevs=home:${projectDirectoryParent},nixstore:/nix/store
@@ -51,7 +51,7 @@ ssh_config=/root/${projectDirectoryName}/benchmark/conf/ssh_config_doctor_cluste
 
 [loadgen]
 fqdn=localhost
-test_iface=enp24s0f1
+test_iface=tap-${username}
 test_iface_addr=0000:18:00.1
 test_iface_mac=B4:96:91:B3:8B:04
 test_iface_ip_net=10.1.0.2/21

benchmark/conf/autotest_localhost.cfg benchmark/conf/uk_localhost.cfg

@@ -2,6 +2,9 @@
 import netaddr
 from pathlib import Path
 import ipaddress
+import getpass
+
+USERNAME: str = getpass.getuser()
 
 class Machine(Enum):
     # Machine types
@@ -41,6 +44,9 @@ class Interface(Enum):
     # VFIO-passthrough to physical NIC for the VM
     VFIO = "vfio"
 
+    # vhost-user device backed by VPP software router
+    VPP = "vpp"
+
     # VMux-passthrough to physical NIC for the VM
     VMUX_PT = "vmux-pt"
 
@@ -66,12 +72,18 @@ def needs_br_tap(self) -> bool:
     def needs_macvtap(self) -> bool:
         return self in [ Interface.MACVTAP ]
 
-    def needs_vfio(self) -> bool:
-        return self in [ Interface.VFIO, Interface.VMUX_PT, Interface.VMUX_DPDK, Interface.VMUX_DPDK_E810, Interface.VMUX_MED ]
+    def needs_vfio(self) -> bool: # vfio on the host
+        return self in [ Interface.VFIO, Interface.VMUX_PT, Interface.VMUX_DPDK, Interface.VMUX_DPDK_E810, Interface.VMUX_MED, Interface.VPP ]
 
     def needs_vmux(self) -> bool:
         return self in [ Interface.VMUX_PT, Interface.VMUX_EMU, Interface.VMUX_DPDK, Interface.VMUX_EMU_E810, Interface.VMUX_MED, Interface.VMUX_DPDK_E810 ]
 
+    def needs_vpp(self) -> bool:
+        return self in [ Interface.VPP ]
+
+    def is_vhost_user(self) -> bool:
+        return self in [ Interface.VPP ]
+
     def is_passthrough(self) -> bool:
         return self in [ Interface.VFIO, Interface.VMUX_PT ]
 
@@ -83,7 +95,7 @@ def guest_driver(self) -> str:
             return "ice"
         if self in [ Interface.BRIDGE_E1000, Interface.VMUX_EMU, Interface.VMUX_DPDK ]:
             return "e1000"
-        if self in [ Interface.BRIDGE, Interface.BRIDGE_VHOST, Interface.MACVTAP ]:
+        if self in [ Interface.BRIDGE, Interface.BRIDGE_VHOST, Interface.MACVTAP, Interface.VPP ]:
             return "virtio-net"
         raise Exception(f"Dont know which guest driver is used with {self}")
 
@@ -172,6 +184,9 @@ def iface_name(tap_name: str, vm_number: int):
         if vm_number == -1: return f"{tap_name[:length]}-"
         return f"{tap_name[:length]}-{vm_number}"
 
+    @staticmethod
+    def vhost_user_sock(vm_number: int):
+        return f"/tmp/vhost-user-{USERNAME}.{vm_number}"
 
     @staticmethod
     def enumerate(enumeratable: str, vm_number: int) -> str: