Skip to content

Ronenroth/configure passthrough numa fixes #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ronenroth-lb wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Ronenroth/configure passthrough numa fixes #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f6cc525
Add suport to create a bridge on top of dhcp interface:x
Dec 8, 2022
e130448
Fix a problem that there might be a collition in the controller slot …
Dec 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 19 additions & 12 deletions vagrant/scripts/configure_passthrough_numa.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,54 +1,61 @@
#!/usr/bin/bash
#!/usr/bin/bash


# Show usage via commandline arguments
usage() {
echo "Usage: ./$0 <vm_name>"
echo "If no vm name is provided, the system will find the running vagrant vms with 'node' in the name and update them"
echo ""
virsh list --state-running
exit
}

function vm_update {
vm=$1
echo "processing VM $vm"
num_of_devices=`virsh dumpxml playground_node00 | grep -c "hostdev mode"`
device_per_numa=`echo $(($num_of_devices/2))`
declare -i num_of_devices=`virsh dumpxml --domain $vm | grep -c "hostdev mode"`
declare -i device_per_numa=`echo $(($num_of_devices/2))`
echo "Found $num_of_devices"
max_slot=`virsh dumpxml --domain $vm | grep bus=\'0x00\' | grep slot | awk '{print $5}' | sort | tail -1 | cut -d \' -f2`
new_slot=`printf "0x%X\n" $(($max_slot+1))`

echo "Adding controllers"
virt-xml $vm --add-device --controller model=pci-expander-bus,type=pci,address.type=pci,address.domain=0x0000,address.bus=0x00,address.slot=0x10,address.function=0x0,index=3
virt-xml $vm --add-device --controller model=pci-expander-bus,type=pci,address.type=pci,address.domain=0x0000,address.bus=0x00,address.slot=0x11,address.function=0x0,index=4
virt-xml $vm --add-device --controller model=pci-expander-bus,type=pci,address.type=pci,address.domain=0x0000,address.bus=0x00,address.slot=${new_slot},address.function=0x0,index=0x03
new_slot=`printf "0x%X" $(($max_slot+4))`
virt-xml $vm --add-device --controller model=pci-expander-bus,type=pci,address.type=pci,address.domain=0x0000,address.bus=0x00,address.slot=${new_slot},address.function=0x0,index=0x04
echo "Num of devices: $num_of_devices"
echo "Devices per numa: $device_per_numa"
bus="0x03"
declare -i slot
for i in $(seq 1 $num_of_devices)
do
slot="$i"
if [[ $i > $device_per_numa ]]
slot=$i
if (( $i > $device_per_numa ))
then
bus="0x04"
slot=`echo $(($i-$device_per_numa))`
slot=`printf "0x%X" $(($i-$device_per_numa))`
fi
echo "Processing device #$i"
echo "Processing device #$i, put on BUS=$bus, slot=$slot"
virt-xml $vm --edit $i --hostdev address.bus=$bus,address.slot=\'0x${slot}\'
done
}

if [[ ! -z "$1" ]] ;then
if [[ $1 == "--help" || $1 == "-h" ]] ; then
usage
virsh list --state-running
else
if [[ `virsh list | grep running | grep -wc $1` -ne "0" ]] ;then
if [[ `virsh list --state-running| grep running | grep -wc $1` -ne "0" ]] ;then
vm_update $1
else
echo "vm $1 seems to be not running or wrong VM name - Exiting"
echo "Current relevant running VMs:"
echo "`virsh list | grep node | grep running | awk '{print $2}'`"
echo "`virsh list --state-running`"
exit
fi
fi
else
for vm in `virsh list | grep node | grep running | awk '{print $2}'` ;do
for vm in `virsh list --state-running |grep running | awk '{print $2}'` ;do
vm_update $vm
done;
fi
77 changes: 48 additions & 29 deletions vagrant/scripts/create_bridge.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ function check_eth {
}

function display_usage {
echo
echo "create a bridge network using a provided interface"
echo "usage:"
echo ""
Expand All @@ -33,11 +34,14 @@ function display_usage {
echo "options:"
echo "bridge_name - our vagrant boxes use br1 as a bridge name."
echo "iface_name - name of the interface we want to use under the bridge."
echo "bridge mode - static or dhcp"
echo "bridge_cidr - subnet each we will issue data IPs for each provisioned VM."
echo "gateway - IP address inside the provided <bridge_cidr>"
echo ""
echo "example:"
echo "./create_bridge.sh br1 ens1f0np0 10.10.230.2/24 10.10.230.1"
echo "For Static IP configuration: ./create_bridge.sh br1 ens1f0np0 static 10.10.230.2/24 10.10.230.1"
echo "For DHCP configuration: ./create_bridge.sh br1 ens1f0np0 dhcp"
echo
}

# check whether user had supplied -h or --help . If yes display usage
Expand All @@ -46,19 +50,25 @@ if [[ ( $@ == "--help") || $@ == "-h" ]]; then
exit 0
fi

# if less than five arguments supplied, display usage
if [ $# -le 3 ]; then
# if less than three arguments supplied, display usage
if [ $# -le 2 ]; then
display_usage
exit 1
fi

install_dependencies


export BRIDGE_NAME=${1}
export BRIDGE_INTERFACE=${2}
export BRIDGE_ADDRESS_CIDR=${3}
export BRIDGE_GATEWAY=${4}
export BRIDGE_MODE=${3}
export BRIDGE_ADDRESS_CIDR=${4}
export BRIDGE_GATEWAY=${5}

if [[ $BRIDGE_MODE != "static" && $BRIDGE_MODE != "dhcp" ]] ;then
printf "\nWrong value for bridge mode, valid values are static or dhcp\n"
display_usage
exit 1
fi

install_dependencies

if check_eth $BRIDGE_INTERFACE; then
echo "The provided link for the bridge is Online!"
Expand All @@ -67,26 +77,30 @@ else
exit 1
fi

# n - ip format validation, m - subnet format validation
n='([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'
m='([0-9]|[12][0-9]|3[012])'

# validate cidr with n & m
if [[ $BRIDGE_ADDRESS_CIDR =~ ^$n(\.$n){3}/$m$ ]]; then
printf '"%s" is a valid CIDR\n' "$BRIDGE_ADDRESS_CIDR"
else
printf 'ERROR: "%s" is not valid CIDR..exiting\n' "$BRIDGE_ADDRESS_CIDR"
exit 1
fi
# n - ip format validation, m - subnet format validation
if [[ $BRIDGE_MODE == "static" ]] ; then
n='([0-9]|[1-9][0-9]|1[0-9][0-9]|2[0-4][0-9]|25[0-5])'
m='([0-9]|[12][0-9]|3[012])'
# CIDR value validation
if [[ $BRIDGE_ADDRESS_CIDR =~ ^$n(\.$n){3}/$m$ ]]; then
printf '"%s" is a valid CIDR\n' "$BRIDGE_ADDRESS_CIDR"
else
printf 'ERROR: "%s" is not valid CIDR..exiting\n' "$BRIDGE_ADDRESS_CIDR"
exit 1
fi

# ip format validation
if [[ $BRIDGE_GATEWAY =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
printf '"%s" is a valid IP\n' "$BRIDGE_GATEWAY"
else
printf 'ERROR: "%s" is not valid gateway address\n' "$BRIDGE_GATEWAY"
exit 1
# ip format validation
if [[ $BRIDGE_GATEWAY =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
printf '"%s" is a valid IP\n' "$BRIDGE_GATEWAY"
else
printf 'ERROR: "%s" is not valid gateway address\n' "$BRIDGE_GATEWAY"
exit 1
fi
fi


set +e
nmcli con show "$BRIDGE_NAME" &>/dev/null
RESULT=$?
Expand All @@ -100,13 +114,18 @@ echo "creating bridge $BRIDGE_NAME"
# start creating the bridge
sudo systemctl stop libvirtd
sudo nmcli con add type bridge ifname $BRIDGE_NAME autoconnect yes con-name $BRIDGE_NAME stp off
sudo nmcli con modify $BRIDGE_NAME ipv4.addresses $BRIDGE_ADDRESS_CIDR ipv4.method manual
sudo nmcli con modify $BRIDGE_NAME ipv4.gateway $BRIDGE_GATEWAY
sudo nmcli con modify $BRIDGE_NAME ipv4.dns $BRIDGE_GATEWAY
if [[ $BRIDGE_MODE == "static" ]] ; then
sudo nmcli con modify $BRIDGE_NAME ipv4.addresses $BRIDGE_ADDRESS_CIDR ipv4.method manual
sudo nmcli con modify $BRIDGE_NAME ipv4.gateway $BRIDGE_GATEWAY
sudo nmcli con modify $BRIDGE_NAME ipv4.dns $BRIDGE_GATEWAY
else
#sudo nmcli con modify $BRIDGE_NAME ipv4.addresses $BRIDGE_ADDRESS_CIDR ipv4.method auto
sudo nmcli con modify $BRIDGE_NAME ipv4.method auto
fi
sudo nmcli con add type bridge-slave autoconnect yes con-name "$BRIDGE_INTERFACE" ifname "$BRIDGE_INTERFACE" master $BRIDGE_NAME
sudo systemctl restart NetworkManager
sudo systemctl restart libvirtd
echo "net.ipv4.ip_forward = 1" | sudo tee /etc/sysctl.d/99-ipforward.conf
sudo sysctl -p /etc/sysctl.d/99-ipforward.conf
sudo nmcli con up $BRIDGE_NAME
sudo nmcli con modify $BRIDGE_NAME connection.autoconnect-slaves 1
sudo nmcli con up $BRIDGE_NAME
sudo systemctl restart NetworkManager
sudo systemctl restart libvirtd