fabric-fpga

name	fabric-fpga
description	Provision and flash Xilinx U280 FPGAs on FABRIC nodes
allowed-tools	["Read","Grep","Glob","Write","Edit","Bash"]

Instructions

When invoked, generate code to provision FPGA nodes on FABRIC and flash bitstreams. There are three main workflows:

1. ESnet — Docker-based SmartNIC firmware using esnet-smartnic-fw
2. NEU — Vivado Lab Tools CLI-based flashing
3. XDMA — Alveo shell-based flashing with xbflash2

Help the user choose the right workflow and generate setup code including slice creation, tool installation, flashing, PCI rescan, and verification.

API Reference

FPGA Component

# Only available FPGA model
node.add_component(model='FPGA_Xilinx_U280', name='fpga1')

Finding Sites with FPGAs

site = fablib.get_random_site(
    filter_function=lambda x: x['fpga_u280_available'] > 0
)

PCI Rescan (After Flashing)

# FABlib method (preferred)
node.rescan_pci(component_name="fpga1")

# Verify devices
stdout, _ = node.execute("lspci | grep Xilinx")

Patterns

FPGA Slice Creation

from fabrictestbed_extensions.fablib.fablib import FablibManager

fablib = FablibManager()

site = fablib.get_random_site(
    filter_function=lambda x: x['fpga_u280_available'] > 0
)

slice = fablib.new_slice(name="fpga-experiment")

node = slice.add_node(
    name="fpga-node",
    site=site,
    cores=8,
    ram=32,
    disk=200,
    image="docker_ubuntu_20",
)
node.add_component(model="FPGA_Xilinx_U280", name="fpga1")
node.add_fabnet()

slice.submit()

ESnet Workflow (Docker-Based)

node = slice.get_node(name="fpga-node")

# Download Docker images and firmware from FABRIC storage
nginx_user = "fpga_tools"
nginx_password = "secret-password"
tools_location = "/home/ubuntu"

downloads = [
    "smartnic-docker-images/smartnic-dpdk-docker.tar.gz",
    "smartnic-docker-images/xilinx-labtools-docker-2023.2_1013_2256.tar.gz",
    "esnet-smartnic-fw/esnet-smartnic-fw.tar.gz",
]

for file in downloads:
    node.execute(
        f"curl -k -u {nginx_user}:{nginx_password} "
        f"https://resources.fabric-testbed.net/fpga-tools/{file} "
        f"> {tools_location}/{file.split('/')[-1]}"
    )

# Load Docker images
node.execute(f"docker load < {tools_location}/smartnic-dpdk-docker.tar.gz")
node.execute(f"docker load < {tools_location}/xilinx-labtools-docker-2023.2_1013_2256.tar.gz")

# Extract firmware
node.execute(f"tar xf {tools_location}/esnet-smartnic-fw.tar.gz -C {tools_location}")

# Download artifact (bitstream)
artifact = "artifacts.au280.p4_only.0.zip"
node.execute(
    f"curl -k -u {nginx_user}:{nginx_password} "
    f"https://resources.fabric-testbed.net/fpga-tools/esnet-artifacts/{artifact} "
    f"> {tools_location}/esnet-smartnic-fw/sn-stack/{artifact}"
)

# Flash the FPGA (takes up to 20 minutes)
stdout, stderr = node.execute(
    "cd esnet-smartnic-fw/sn-stack/ && "
    "docker compose --profile smartnic-flash run --rm smartnic-flash-rescue && "
    "docker compose down -v --remove-orphans"
)

# PCI rescan after flashing
node.rescan_pci(component_name="fpga1")

# Verify — should see Device 903f and Device 913f
stdout, _ = node.execute("lspci | grep Xilinx")
print(stdout)

# Enable CMAC interfaces
node.execute(
    "cd esnet-smartnic-fw/sn-stack/ && "
    "docker compose up -d && "
    "sleep 10 && "
    "docker compose exec smartnic-fw sn-cli cmac enable"
)

# Check CMAC status — expect "PHY UP -> UP"
stdout, _ = node.execute(
    "cd esnet-smartnic-fw/sn-stack/ && "
    "docker compose exec smartnic-fw sn-cli cmac status"
)
print(stdout)

NEU Workflow (Vivado CLI)

node = slice.get_node(name="fpga-node")

nginx_user = "fpga_tools"
nginx_password = "secret-password"
tools_location = "/home/ubuntu/fpga-tools"
vivado_install_dir = "/tools/Xilinx"
version = "2023.2_1013_2256"
version_major = "2023.2"

# Download Vivado Lab Tools
node.execute(f"mkdir -p {tools_location}")
node.execute(
    f"curl -k -u {nginx_user}:{nginx_password} "
    f"https://resources.fabric-testbed.net/fpga-tools/Vivado/Vivado_Lab_Lin_{version}.tar.gz "
    f"> {tools_location}/Vivado_Lab_Lin_{version}.tar.gz"
)

# Extract and install
node.execute(f"tar xf {tools_location}/Vivado_Lab_Lin_{version}.tar.gz -C {tools_location}")
node.execute(
    f"sudo {tools_location}/Vivado_Lab_Lin_{version}/xsetup "
    f'--agree 3rdPartyEULA,XilinxEULA --batch Install '
    f'--edition "Vivado Lab Edition (Standalone)" '
    f"--location {vivado_install_dir}"
)

# Clone required repos
node.execute(
    f"cd {tools_location} && "
    "git clone --branch fpga-flash-boot-from-config-mem "
    "https://github.com/mcevik0/P4OpenNIC_Public.git"
)

# Upload bitstream artifact (.mcs file)
artifact = "addhdr_jumbo.mcs"
bitfile_location = f"{tools_location}/bitfiles"
node.execute(f"mkdir -p {bitfile_location}")
# Upload your .mcs file to bitfile_location

# Flash using Vivado
node.execute(
    f"source {vivado_install_dir}/Vivado_Lab/{version_major}/settings64.sh && "
    f"export JTAG_ID=`cat /tmp/fpga-jtag.id` && "
    f"cd {tools_location}/P4OpenNIC_Public/FABRIC-P4/scripts && "
    f"./program_flash.sh {bitfile_location}/{artifact} au280 $JTAG_ID"
)

# PCI rescan
node.rescan_pci(component_name="fpga1")
stdout, _ = node.execute("lspci | grep Xilinx")
print(stdout)

XDMA Workflow (xbflash2)

node = slice.get_node(name="fpga-node")

nginx_user = "fpga_tools"
nginx_password = "secret-password"
tools_location = "/home/ubuntu/fpga-tools"

# Download and install XRT and xbflash2
xrt_package = "alveo-packages/xrt_202320.2.16.204_20.04-amd64-xrt.deb"
xbflash2_package = "alveo-packages/xrt_202210.2.13.466_20.04-amd64-xbflash2.deb"

for pkg in [xrt_package, xbflash2_package]:
    filename = pkg.split("/")[-1]
    node.execute(
        f"curl -k -u {nginx_user}:{nginx_password} "
        f"https://resources.fabric-testbed.net/fpga-tools/{pkg} "
        f"> {tools_location}/{filename}"
    )

node.execute(f"sudo dpkg -i {tools_location}/xrt_*.deb || sudo apt-get -f install -y")

# Get FPGA BDF address
fpga_bdf = "0000:1f:00.0"  # Default, verify with lspci

# Flash with xbflash2
artifact = "xilinx_u280_gen3x16_xdma_1_202211_1_1691631390_xdma-shell.mcs"
stdout, stderr = node.execute(
    f"yes | sudo xbflash2 program --spi "
    f"--image {tools_location}/{artifact} "
    f"-d {fpga_bdf.replace('0000:', '')}"
)

# PCI rescan
node.rescan_pci(component_name="fpga1")
stdout, _ = node.execute("lspci | grep Xilinx")
print(stdout)

IOMMU and Hugepages Setup (For DPDK on FPGA)

# Configure GRUB for IOMMU and hugepages
node.execute(
    'sudo sed -i '
    "'s/GRUB_CMDLINE_LINUX=\"\"/GRUB_CMDLINE_LINUX=\"amd_iommu=on iommu=pt "
    'default_hugepagesz=1G hugepagesz=1G hugepages=8"/'
    "' /etc/default/grub"
)
node.execute("sudo update-grub")

# Reboot required
node.execute("sudo reboot")
slice.wait_ssh(timeout=360, interval=10, progress=True)

# Enable unsafe_noiommu_mode for VFIO
node.execute(
    "echo 1 | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode"
)

Important Notes

• Never interrupt flashing — can brick the FPGA
• Always flash before use — previous experiments may leave FPGA in unknown state
• Recommended node specs: 8 cores, 32 GB RAM, 200 GB disk, docker_ubuntu_20 image
• Use node.rescan_pci() after every flash operation
• ESnet expected PCI devices: 903f and 913f
• NEU expected PCI devices: 903f
• XDMA expected PCI devices: 500c (mgmt) and 500d (user)
• Flash time: Up to 20 minutes for persistent flash
• FPGA tools are downloaded from resources.fabric-testbed.net with credentials