ci: nbd provisioning groundwork for Blacksmith (KVM jobs blocked on kernel >= 6.7) #1
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Blacksmith Kernel Probe | |
| # Temporary diagnostic workflow: establishes whether Blacksmith's custom VM | |
| # kernel can support the orchestrator/integration test requirements (nbd | |
| # module, KVM, tun, vsock, uffd, hugepages). Delete once the verdict is in. | |
| on: | |
| pull_request: | |
| paths: | |
| - ".github/workflows/blacksmith-kernel-probe.yml" | |
| permissions: | |
| contents: read | |
| jobs: | |
| probe: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: | |
| - blacksmith-8vcpu-ubuntu-2404 | |
| - blacksmith-32vcpu-ubuntu-2404 | |
| - blacksmith-4vcpu-ubuntu-2404-arm | |
| - blacksmith-2vcpu-ubuntu-2204 | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Kernel and device diagnostics | |
| run: | | |
| set -x | |
| uname -a | |
| cat /proc/version || true | |
| echo "--- /lib/modules ---" | |
| ls -la /lib/modules/ || true | |
| ls /lib/modules/"$(uname -r)"/ 2>/dev/null || echo "no modules dir for running kernel" | |
| find /lib/modules -name 'nbd.ko*' 2>/dev/null || true | |
| grep -c . /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "no modules.builtin" | |
| grep nbd /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "nbd not builtin (or no list)" | |
| echo "--- kernel config ---" | |
| CONFIG_SRC="" | |
| if [ -e /proc/config.gz ]; then CONFIG_SRC=/proc/config.gz | |
| elif [ -e "/boot/config-$(uname -r)" ]; then CONFIG_SRC="/boot/config-$(uname -r)" | |
| fi | |
| echo "config source: ${CONFIG_SRC:-NONE}" | |
| if [ -n "$CONFIG_SRC" ]; then | |
| zgrep -hE '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_SIG_ALL|MODULE_FORCE_LOAD|MODULE_COMPRESS|BLK_DEV_NBD|BLK_DEV_LOOP|TUN|VSOCKETS|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|KVM_INTEL|KVM_AMD|IKHEADERS|IKCONFIG|NF_TABLES|IP_NF_IPTABLES|NETFILTER_XTABLES|BRIDGE|VETH|FUSE_FS|OVERLAY_FS|SQUASHFS|LOCALVERSION)[ =]' "$CONFIG_SRC" 2>/dev/null \ | |
| || { gzip -t "$CONFIG_SRC" 2>/dev/null && zcat "$CONFIG_SRC" | grep -E '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|TUN|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|IKHEADERS)[ =]'; } || true | |
| fi | |
| echo "--- devices ---" | |
| ls -la /dev/kvm /dev/net/tun /dev/vhost-vsock /dev/fuse /dev/nbd0 /dev/loop0 /dev/loop-control 2>&1 || true | |
| echo "--- loaded modules ---" | |
| lsmod | head -30 || cat /proc/modules | head -30 || true | |
| echo "--- kheaders availability ---" | |
| sudo modprobe kheaders 2>&1 || true | |
| ls -la /sys/kernel/kheaders.tar.xz 2>&1 || true | |
| echo "--- headers / build tree ---" | |
| ls -la /lib/modules/"$(uname -r)"/build 2>&1 || true | |
| ls /usr/src/ 2>&1 || true | |
| apt-get -s install "linux-headers-$(uname -r)" 2>&1 | tail -3 || true | |
| echo "--- host features ---" | |
| nproc; free -h; df -h / /tmp | |
| sudo sysctl vm.unprivileged_userfaultfd || true | |
| mount | grep -iE 'hugetlbfs|cgroup2' || true | |
| docker info --format '{{.Driver}} kernel={{.KernelVersion}}' 2>/dev/null || true | |
| - name: Attempt nbd module build from kernel.org source | |
| id: build | |
| continue-on-error: true | |
| run: | | |
| set -euxo pipefail | |
| T0=$(date +%s) | |
| KVER="$(uname -r)" | |
| BASE="${KVER%%-*}" | |
| # Bail out early if modprobe already works (e.g. image got fixed) | |
| if sudo modprobe nbd nbds_max=256 2>/dev/null; then | |
| echo "modprobe worked natively, nothing to build" | |
| exit 0 | |
| fi | |
| # Kernel config is required to build a compatible module | |
| if [ -e /proc/config.gz ]; then | |
| CONFIG_CMD="zcat /proc/config.gz" | |
| elif [ -e "/boot/config-$KVER" ]; then | |
| CONFIG_CMD="cat /boot/config-$KVER" | |
| else | |
| echo "::error::no kernel config available (no /proc/config.gz, no /boot/config)" | |
| exit 1 | |
| fi | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev dwarves | |
| cd /tmp | |
| curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz" | |
| tar -xf "linux-${BASE}.tar.xz" | |
| cd "linux-${BASE}" | |
| eval "$CONFIG_CMD" > .config | |
| # Force nbd to build as a module regardless of the shipped config | |
| ./scripts/config -m BLK_DEV_NBD | |
| # A signing key we don't have; disable so modpost doesn't try to sign | |
| ./scripts/config --disable MODULE_SIG_ALL || true | |
| ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true | |
| ./scripts/config --set-str MODULE_SIG_KEY "" || true | |
| make olddefconfig | |
| echo "--- effective module-relevant config ---" | |
| grep -E '^CONFIG_(MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|LOCALVERSION)[ =]' .config || true | |
| T1=$(date +%s); echo "setup took $((T1-T0))s" | |
| make -j"$(nproc)" modules_prepare | |
| T2=$(date +%s); echo "modules_prepare took $((T2-T1))s" | |
| make -j"$(nproc)" M=drivers/block modules | |
| T3=$(date +%s); echo "module build took $((T3-T2))s" | |
| modinfo drivers/block/nbd.ko | head -20 | |
| echo "--- vermagic comparison ---" | |
| modinfo -F vermagic drivers/block/nbd.ko || true | |
| echo "running kernel: $KVER" | |
| if ! sudo insmod drivers/block/nbd.ko nbds_max=256; then | |
| echo "plain insmod failed, dmesg:" | |
| sudo dmesg | tail -20 | |
| # Retry via modprobe --force (strips vermagic/version checks; | |
| # needs CONFIG_MODULE_FORCE_LOAD=y in the running kernel) | |
| sudo mkdir -p "/lib/modules/$KVER/kernel/drivers/block" | |
| sudo cp drivers/block/nbd.ko "/lib/modules/$KVER/kernel/drivers/block/" | |
| sudo depmod -a || true | |
| sudo modprobe --force nbd nbds_max=256 || { | |
| echo "forced modprobe also failed, dmesg:" | |
| sudo dmesg | tail -20 | |
| exit 1 | |
| } | |
| fi | |
| T4=$(date +%s); echo "TOTAL nbd provisioning: $((T4-T0))s" | |
| - name: Verdict | |
| run: | | |
| set -x | |
| echo "=== VERDICT for ${{ matrix.runner }} ===" | |
| NBD_OK=false; KVM_OK=false; TUN_OK=false; VSOCK_OK=false | |
| [ -b /dev/nbd0 ] && [ -b /dev/nbd255 ] && NBD_OK=true | |
| [ -e /dev/kvm ] && KVM_OK=true | |
| [ -e /dev/net/tun ] && TUN_OK=true | |
| [ -e /dev/vhost-vsock ] && VSOCK_OK=true | |
| sudo dmesg | grep -iE 'nbd|taint' | tail -5 || true | |
| echo "NBD=$NBD_OK KVM=$KVM_OK TUN=$TUN_OK VSOCK=$VSOCK_OK" | |
| # nbd is the make-or-break requirement on every runner | |
| $NBD_OK |