From 197e5ae8ef990888b9eaac1ada421370b2306a48 Mon Sep 17 00:00:00 2001 From: Erika Hunhoff Date: Tue, 27 Jun 2023 19:48:26 -0700 Subject: [PATCH] Add compile flag for affinity shmem, fix build + clippy errors --- kernel/Cargo.toml | 2 + kernel/src/arch/x86_64/mod.rs | 7 +++- kernel/src/pci.rs | 2 +- kernel/tests/s11_rackscale_benchmarks.rs | 52 ++++++++++++++++-------- kernel/testutils/src/helpers.rs | 11 +++-- 5 files changed, 50 insertions(+), 24 deletions(-) diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 91c4ed40b..4fc200266 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -113,6 +113,8 @@ integration-test = ["shmem", "ethernet"] smoke = [] # For rackscale tests, runs baseline NrOS. This causes the tests to take longer. baseline = [] +# For rackscale tests, use shmem allocated with numa affinity. This requires pre-configuring the host. +affinity-shmem = [] # baremetal: Compile benchmarks for running on bare-metal. baremetal = [] # pre-alloc guest memory: For benchmark sensitive to VM exits. diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index b4ad6fd4d..7e37440eb 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -307,8 +307,11 @@ fn _start(argc: isize, _argv: *const *const u8) -> isize { // form by klogger::init above, but now we do it for more ports) debug::init(); - use crate::transport::shmem::SHMEM_INITIALIZED; - lazy_static::initialize(&SHMEM_INITIALIZED); + #[cfg(feature = "rackscale")] + { + use crate::transport::shmem::SHMEM_INITIALIZED; + lazy_static::initialize(&SHMEM_INITIALIZED); + } // Parse memory map provided by UEFI, create an initial emergency memory // manager with a little bit of memory so we can do some early allocations. diff --git a/kernel/src/pci.rs b/kernel/src/pci.rs index 9876d5ed3..4afcfb714 100644 --- a/kernel/src/pci.rs +++ b/kernel/src/pci.rs @@ -59,7 +59,7 @@ pub(crate) fn claim_devices(vendor_id: u16, device_id: u16) -> Vec { } } } - return devices; + devices } pub(crate) fn init() { diff --git a/kernel/tests/s11_rackscale_benchmarks.rs b/kernel/tests/s11_rackscale_benchmarks.rs index 3c777b067..783e6e37a 100644 --- a/kernel/tests/s11_rackscale_benchmarks.rs +++ b/kernel/tests/s11_rackscale_benchmarks.rs @@ -141,8 +141,13 @@ fn rackscale_fxmark_benchmark(is_shmem: bool) { ); let (shmem_socket, shmem_file) = get_shmem_names(None); + let shmem_affinity = if cfg!(feature = "affinity-shmem") { + Some(0) + } else { + None + }; let mut shmem_server = - spawn_shmem_server(&shmem_socket, &shmem_file, shmem_size, Some(0)) + spawn_shmem_server(&shmem_socket, &shmem_file, shmem_size, shmem_affinity) .expect("Failed to start shmem server"); let mut cmdline_baseline = @@ -250,14 +255,15 @@ fn rackscale_fxmark_benchmark(is_shmem: bool) { let mut shmem_sockets = Vec::new(); let mut shmem_servers = Vec::new(); for i in 0..(num_clients + 1) { + let shmem_affinity = if cfg!(feature = "affinity-shmem") { + Some(placement_cores[i].0) + } else { + None + }; let (shmem_socket, shmem_file) = get_shmem_names(Some(i)); - let shmem_server = spawn_shmem_server( - &shmem_socket, - &shmem_file, - shmem_size, - Some(placement_cores[i].0), - ) - .expect("Failed to start shmem server"); + let shmem_server = + spawn_shmem_server(&shmem_socket, &shmem_file, shmem_size, shmem_affinity) + .expect("Failed to start shmem server"); shmem_sockets.push(shmem_socket); shmem_servers.push(shmem_server); } @@ -570,9 +576,18 @@ fn rackscale_vmops_benchmark(is_shmem: bool, benchtype: VMOpsBench) { ); let (shmem_socket, shmem_file) = get_shmem_names(None); - let mut shmem_server = - spawn_shmem_server(&shmem_socket, &shmem_file, baseline_shmem_size, Some(0)) - .expect("Failed to start shmem server"); + let shmem_affinity = if cfg!(feature = "affinity-shmem") { + Some(0) + } else { + None + }; + let mut shmem_server = spawn_shmem_server( + &shmem_socket, + &shmem_file, + baseline_shmem_size, + shmem_affinity, + ) + .expect("Failed to start shmem server"); let baseline_cmdline = format!("initargs={}", cores); let baseline_file_name = file_name.clone(); @@ -693,14 +708,15 @@ fn rackscale_vmops_benchmark(is_shmem: bool, benchtype: VMOpsBench) { let mut shmem_sockets = Vec::new(); let mut shmem_servers = Vec::new(); for i in 0..(num_clients + 1) { + let shmem_affinity = if cfg!(feature = "affinity-shmem") { + Some(placement_cores[i].0) + } else { + None + }; let (shmem_socket, shmem_file) = get_shmem_names(Some(i)); - let shmem_server = spawn_shmem_server( - &shmem_socket, - &shmem_file, - shmem_size, - Some(placement_cores[i].0), - ) - .expect("Failed to start shmem server"); + let shmem_server = + spawn_shmem_server(&shmem_socket, &shmem_file, shmem_size, shmem_affinity) + .expect("Failed to start shmem server"); shmem_sockets.push(shmem_socket); shmem_servers.push(shmem_server); } diff --git a/kernel/testutils/src/helpers.rs b/kernel/testutils/src/helpers.rs index 34f3ca83d..864f3a988 100644 --- a/kernel/testutils/src/helpers.rs +++ b/kernel/testutils/src/helpers.rs @@ -26,7 +26,7 @@ pub const DHCP_ACK_MATCH_NRK2: &'static str = "DHCPACK on 172.31.0.11 to 56:b4:4 /// Default shmem region size (in MB) pub const SHMEM_SIZE: usize = 1024; /// Created by `hugeadm --create-global-mounts` -const SHMEM_PATH: &'static str = "/var/lib/hugetlbfs/global/pagesize-2MB"; +const SHMEM_AFFINITY_PATH: &'static str = "/var/lib/hugetlbfs/global/pagesize-2MB"; /// Delay between invoking version of nrk in rackscale tests pub const CLIENT_BUILD_DELAY: u64 = 5_000; @@ -52,15 +52,20 @@ pub fn setup_network(num_nodes: usize) { } pub fn get_shmem_names(id: Option) -> (String, String) { + let shmem_path = if cfg!(feature = "affinity-shmem") { + SHMEM_AFFINITY_PATH + } else { + "." + }; if let Some(shmemid) = id { ( format!("ivshmem-socket{}", shmemid), - format!("{}/ivshmem-file{}", SHMEM_PATH, shmemid), + format!("{}/ivshmem-file{}", shmem_path, shmemid), ) } else { ( format!("ivshmem-socket{}", ""), - format!("{}/ivshmem-file{}", SHMEM_PATH, ""), + format!("{}/ivshmem-file{}", shmem_path, ""), ) } }