fabric-benchmarking

name	fabric-benchmarking
description	Run iPerf3 network performance benchmarks with optional CPU pinning and SmartNIC optimization
allowed-tools	["Read","Grep","Glob","Write","Edit","Bash"]

Instructions

When invoked, generate code for network performance benchmarking on FABRIC using iPerf3. Three tiers:

1. Basic — iPerf3 over FABNet with Docker containers
2. Optimized — CPU pinning, NUMA tuning, and host_tune.sh
3. SmartNIC — Dedicated ConnectX NICs with full hardware optimization

Help the user choose the right level and generate complete code.

API Reference

Performance Tuning

node.pin_cpu(component_name='nic1')  # Pin vCPUs to NIC's NUMA node
node.numa_tune()                      # Pin memory to same NUMA node
node.os_reboot()                      # Reboot to apply

Post-Reboot Reconfiguration

slice.wait_ssh()
for node in slice.get_nodes():
    node.config()  # Re-configure network interfaces

Patterns

Basic iPerf3 (FABNet + Docker)

from fabrictestbed_extensions.fablib.fablib import FablibManager

fablib = FablibManager()

[site1, site2] = fablib.get_random_sites(count=2)

slice = fablib.new_slice(name="iperf3-basic")

node1 = slice.add_node(
    name="server",
    site=site1,
    cores=4,
    ram=16,
    disk=50,
    image="default_rocky_9",
)
node1.add_fabnet()
node1.add_post_boot_upload_directory('node_tools', '.')
node1.add_post_boot_execute('sudo node_tools/host_tune.sh')
node1.add_post_boot_execute('node_tools/enable_docker.sh {{ _self_.image }}')
node1.add_post_boot_execute(
    'docker pull fabrictestbed/slice-vm-rocky8-multitool:0.0.2'
)

node2 = slice.add_node(
    name="client",
    site=site2,
    cores=4,
    ram=16,
    disk=50,
    image="default_rocky_9",
)
node2.add_fabnet()
node2.add_post_boot_upload_directory('node_tools', '.')
node2.add_post_boot_execute('sudo node_tools/host_tune.sh')
node2.add_post_boot_execute('node_tools/enable_docker.sh {{ _self_.image }}')
node2.add_post_boot_execute(
    'docker pull fabrictestbed/slice-vm-rocky8-multitool:0.0.2'
)

slice.submit()

server = slice.get_node(name="server")
client = slice.get_node(name="client")

server_addr = server.get_interface(
    network_name=f"FABNET_IPv4_{server.get_site()}"
).get_ip_addr()

# Start iPerf3 server (single test mode)
server.execute(
    "docker run -d --rm --network host "
    "fabrictestbed/slice-vm-rocky8-multitool:0.0.2 iperf3 -s -1"
)

# Run iPerf3 client (4 parallel streams, 30 sec, 10 sec omit)
stdout, stderr = client.execute(
    f"docker run --rm --network host "
    f"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 "
    f"iperf3 -c {server_addr} -P 4 -t 30 -i 10 -O 10"
)
print(stdout)

Optimized iPerf3 (CPU Pinning + NUMA)

[site1, site2] = fablib.get_random_sites(count=2)

slice = fablib.new_slice(name="iperf3-optimized")

for name, site in [("server", site1), ("client", site2)]:
    node = slice.add_node(
        name=name,
        site=site,
        cores=8,
        ram=32,
        disk=50,
        image="docker_rocky_8",
    )
    node.add_fabnet()
    node.add_post_boot_upload_directory('node_tools', '.')
    node.add_post_boot_execute('sudo node_tools/host_tune.sh')
    node.add_post_boot_execute('node_tools/enable_docker.sh {{ _self_.image }}')

slice.submit()

# Apply CPU pinning and NUMA tuning
for node in slice.get_nodes():
    node.pin_cpu(component_name='nic1')
    node.numa_tune()

# Reboot to apply
for node in slice.get_nodes():
    node.os_reboot()

slice.wait_ssh()

# Re-configure network interfaces after reboot
for node in slice.get_nodes():
    node.config()
    node.execute("sudo systemctl enable docker && sudo systemctl start docker")

server = slice.get_node(name="server")
client = slice.get_node(name="client")

server_addr = server.get_interface(
    network_name=f"FABNET_IPv4_{server.get_site()}"
).get_ip_addr()

# Run optimized benchmark
server.execute(
    "docker run -d --rm --network host "
    "fabrictestbed/slice-vm-rocky8-multitool:0.0.2 iperf3 -s -1"
)

stdout, _ = client.execute(
    f"docker run --rm --network host "
    f"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 "
    f"iperf3 -c {server_addr} -P 8 -t 60 -i 10 -O 10"
)
print(stdout)

SmartNIC iPerf3 (Dedicated ConnectX NICs)

[site1, site2] = fablib.get_random_sites(
    count=2,
    filter_function=lambda x: x['nic_connectx_5_available'] > 0,
)

slice = fablib.new_slice(name="iperf3-smartnic")

net = slice.add_l3network(name="data-net", type="IPv4")

for name, site in [("server", site1), ("client", site2)]:
    node = slice.add_node(
        name=name,
        site=site,
        cores=16,
        ram=32,
        disk=50,
        image="docker_rocky_8",
    )
    node.add_fabnet()
    iface = node.add_component(model="NIC_ConnectX_5", name="nic1").get_interfaces()[0]
    iface.set_mode("auto")
    net.add_interface(iface)

    node.add_post_boot_upload_directory('node_tools', '.')
    node.add_post_boot_execute('sudo node_tools/host_tune.sh')
    node.add_post_boot_execute('node_tools/enable_docker.sh {{ _self_.image }}')

slice.submit()

# Apply hardware optimization
for node in slice.get_nodes():
    node.pin_cpu(component_name='nic1')
    node.numa_tune()
    node.os_reboot()

slice.wait_ssh()

for node in slice.get_nodes():
    node.config()
    node.execute("sudo systemctl enable docker && sudo systemctl start docker")

server = slice.get_node(name="server")
client = slice.get_node(name="client")

# Use dedicated NIC interface for benchmark
server_addr = server.get_interface(network_name="data-net").get_ip_addr()

server.execute(
    "docker run -d --rm --network host "
    "fabrictestbed/slice-vm-rocky8-multitool:0.0.2 iperf3 -s -1"
)

stdout, _ = client.execute(
    f"docker run --rm --network host "
    f"fabrictestbed/slice-vm-rocky8-multitool:0.0.2 "
    f"iperf3 -c {server_addr} -P 16 -t 60 -i 10 -O 10"
)
print(stdout)

Important Notes

• host_tune.sh: Applies kernel-level network tuning (buffer sizes, TCP settings)
• CPU pinning: pin_cpu() aligns vCPUs with NIC's NUMA node for lowest latency
• NUMA tuning: numa_tune() pins memory to same NUMA node as vCPUs
• After reboot: Always slice.wait_ssh() then node.config() to restore interfaces
• Docker restart: Docker may not auto-start after reboot on some images
• iPerf3 flags: -P parallel streams, -t duration, -O omit warmup seconds, -i interval
• NIC_Basic: ~10 Gbps; ConnectX-5: ~25 Gbps; ConnectX-6: ~100 Gbps