libafl-qemu: "Fast" system-mode snapshots trigger an assertion failure

[langston-barrett]

I'm trying to fuzz an EDK II image. However, the "fast" system-mode snapshots seem to always trigger an assertion failure. Unfortunately, I can't share the compiled artifacts, but I'd imagine this could be reproduced with another system-mode target.

// cargo init
// cargo add --no-default-features --features=systemmode --git https://github.com/AFLplusplus/LibAFL libafl_qemu
// cargo run

use std::error;

const QEMU_FLAGS: &[&str] = &[
    "-machine",
    "q35",
    "-kernel",
    "edk2/bzImage",
    "-append",
    "'rootwait root=/dev/vda console=tty1 console=ttyS0 keep_bootcon'",
    "-drive",
    "if=none,format=qcow2,file=snaps.qcow2",
    "-drive",
    "file=edk2/rootfs.ext2,if=virtio,format=raw,readonly=on",
    "-global",
    "driver=cfi.pflash01,property=secure,value=on",
    "-drive",
    "if=pflash,format=raw,unit=0,file=edk2/OVMF_CODE.fd,readonly=on",
    "-drive",
    "if=pflash,format=raw,unit=1,file=edk2/OVMF_VARS.fd",
    "-smp",
    "2",
    "-m",
    "4G",
    "-bios",
    "edk2/OVMF.fd",
    "-fw_cfg",
    "name=opt/org.tianocore/X-Cpuhp-Bugcheck-Override,string=yes",
    "-serial",
    "file:serial.log",
    "-S",
    "-nodefaults",
    "-vga",
    "none",
    "-nographic",
    "-snapshot",
];

pub fn read_u64(qemu: &libafl_qemu::Qemu, addr: libafl_qemu::GuestAddr) -> u64 {
    let mut val_buf = [0; 8];
    qemu.read_mem(addr, &mut val_buf).unwrap();
    u64::from_le_bytes(val_buf)
}

fn run_to(
    qemu: &libafl_qemu::Qemu,
    addr: libafl_qemu::GuestAddr,
) -> Result<(), libafl_qemu::QemuExitError> {
    qemu.set_breakpoint(addr);
    unsafe { qemu.run() }.unwrap();
    qemu.remove_breakpoint(addr);
    Ok(())
}

fn main() -> Result<(), Box<dyn error::Error>> {
    let mut args = vec!["qemu".to_owned()];
    args.extend(QEMU_FLAGS.iter().map(|s| (*s).to_owned()));
    let qemu = libafl_qemu::Qemu::init(args.as_slice())?;

    let entry = 0x0007FA536BE;
    println!("Running to entrypoint ({:#x})...", entry);
    run_to(&qemu, entry).unwrap();

    let snap = qemu.create_fast_snapshot(true);
    unsafe { qemu.restore_fast_snapshot(snap) };
    Ok(())
}

Output:

Could not open option rom 'kvmvapic.bin': No such file or directory
Could not open option rom 'linuxboot_dma.bin': No such file or directory
Running to entrypoint (0x7fa536be)...
libafl-snapshots: ../system/memory.c:2655: memory_region_add_subregion_common: Assertion `!subregion->container' failed.
zsh: abort (core dumped)  cargo -q run

[rmalmain]

thank you for the report.
do you mind posting the stacktrace when the assert gets triggered?

[langston-barrett]

@rmalmain Here's what GDB says:

gdb -q --batch -ex 'run' -ex 'bt' --args ./target/debug/libafl-snapshots

libafl-snapshots: ../system/memory.c:2655: memory_region_add_subregion_common: Assertion `!subregion->container' failed.

Thread 1 "libafl-snapshot" received signal SIGABRT, Aborted.
0x00007ffff7b2e7dc in __pthread_kill_implementation () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#0  0x00007ffff7b2e7dc in __pthread_kill_implementation () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#1  0x00007ffff7adc516 in raise () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#2  0x00007ffff7ac4935 in abort () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#3  0x00007ffff7ac4859 in __assert_fail_base.cold () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#4  0x00007ffff7ad49f6 in __assert_fail () from /nix/store/3dyw8dzj9ab4m8hv5dpyx7zii8d0w6fi-glibc-2.39-52/lib/libc.so.6
#5  0x0000555555be1a71 in memory_region_add_subregion_common (mr=<optimized out>, offset=<optimized out>, subregion=<optimized out>) at ../system/memory.c:2655
#6  memory_region_add_subregion_common (mr=<optimized out>, offset=<optimized out>, subregion=<optimized out>) at ../system/memory.c:2649
#7  0x0000555555b92239 in ich9_lpc_rcba_update (rcba_old=0, lpc=0x555557acc0f0) at ../hw/isa/lpc_ich9.c:521
#8  ich9_lpc_post_load (opaque=0x555557acc0f0, version_id=<optimized out>) at ../hw/isa/lpc_ich9.c:547
#9  0x0000555555e18782 in vmstate_load_state (f=0x5555571de5d0, vmsd=<optimized out>, opaque=0x555557acc0f0, version_id=<optimized out>) at ../migration/vmstate.c:186
#10 0x0000555555a8e925 in qemu_loadvm_section_start_full (f=f@entry=0x5555571de5d0, type=type@entry=4 '\004', mis=<optimized out>) at ../migration/savevm.c:2612
#11 0x0000555555a929eb in qemu_loadvm_state_main (f=f@entry=0x5555571de5d0, mis=0x5555573457a0) at ../migration/savevm.c:2867
#12 0x0000555555a94794 in qemu_load_device_state (f=f@entry=0x5555571de5d0) at ../migration/savevm.c:3011
#13 0x0000555555bd6f35 in device_restore_all (dss=<optimized out>) at ../libafl/syx-snapshot/device-save.c:108
#14 0x0000555555bd7f09 in syx_snapshot_root_restore (snapshot=0x5555582afff0) at ../libafl/syx-snapshot/syx-snapshot.c:728
#15 0x000055555585a82f in libafl_qemu::qemu::Qemu::restore_fast_snapshot (self=0x7fffffff6cf7, snapshot=0x5555582afff0) at src/qemu/systemmode.rs:263
#16 0x00005555558598f0 in libafl_snapshots::main () at src/main.rs:69

[rmalmain]

can you print the result of qemu.list_devices() please?

[langston-barrett]

Sure! Here they are right after Qemu::init:

["timer", "cpu_common", "cpu", "kvm-tpr-opt", "apic", "cpu_common", "cpu", "apic", "pflash_cfi01", "pflash_cfi01", "fw_cfg", "0000:00:00.0/mch", "PCIHost", "PCIBUS", "dma", "dma", "mc146818rtc", "0000:00:1f.0/ICH9LPC", "i8259", "i8259", "ioapic", "hpet", "i8254", "pcspk", "serial", "ps2kbd", "ps2mouse", "pckbd", "vmmouse", "port92", "0000:00:1f.2/ich9_ahci", "i2c_bus", "0000:00:1f.3/ich9_smb", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "smbus-eeprom", "0000:00:01.0/virtio-blk", "acpi_build"]

[rmalmain]

can you apply this patch to qemu-libafl-bridge locally and check if it solves the problem?

diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c
index bd727b2320..b2d30e4719 100644
--- a/hw/isa/lpc_ich9.c
+++ b/hw/isa/lpc_ich9.c
@@ -517,7 +517,7 @@ static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rcba_old)
     if (rcba_old & ICH9_LPC_RCBA_EN) {
         memory_region_del_subregion(get_system_memory(), &lpc->rcrb_mem);
     }
-    if (rcba & ICH9_LPC_RCBA_EN) {
+    if (rcba & ICH9_LPC_RCBA_EN && !lpc->rcrb_mem.container) {
         memory_region_add_subregion_overlap(get_system_memory(),
                                             rcba & ICH9_LPC_RCBA_BA_MASK,
                                             &lpc->rcrb_mem, 1);

[langston-barrett]

Huh, I must be doing something wrong. I applied your patch here and then modified LibAFL here and then modified my Cargo.toml like so:

[dependencies.libafl_qemu]
git = "https://github.com/langston-barrett/LibAFL"
rev = "ad612461a8e6dde6d9be31345e32a1f7cb1a75e6"
default-features = false
features = ["systemmode"]

But when I run cargo build, I get:

  ERROR: unknown option --disable-tests
  Try '/home/langston/code/tmp/libafl-sys-snapshots/target/debug/qemu-libafl-bridge/configure --help' for more information

  --- stderr
  thread 'main' panicked at /home/langston/.cargo/git/checkouts/libafl-f15b1d9e1d39cd94/ad61246/libafl_qemu/libafl_qemu_build/src/build.rs:412:9:
  Configure didn't finish successfully

[samcowger]

A possible cause is that your qemu-libafl-bridge fork is out of date. Your patch commit, langston-barrett/qemu-libafl-bridge@d99f038, is based on AFLplusplus/qemu-libafl-bridge@0dc52ed, which appears to be vintage 2023.

[rmalmain]

most likely the case yeah.
i just tried to compile it on my side and it doesn't seem to cause any compilation issue.

[langston-barrett]

Good catch! Yes, that patch appears to fix the assertion failure.