/*
 * bhyve_command.c: bhyve command generation
 *
 * Copyright (C) 2014 Roman Bogorodskiy
 * Copyright (C) 2025 The FreeBSD Foundation
 * Copyright (C) 2024-2025 Future Crew, LLC
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.  If not, see
 * <http://www.gnu.org/licenses/>.
 *
 */

#include <config.h>

#include "bhyve_capabilities.h"
#include "bhyve_command.h"
#include "bhyve_domain.h"
#include "bhyve_conf.h"
#include "bhyve_driver.h"
#include "domain_validate.h"
#include "datatypes.h"
#include "viralloc.h"
#include "virfile.h"
#include "virstring.h"
#include "virlog.h"
#include "virnetdev.h"
#include "virnetdevbridge.h"
#include "virnetdevtap.h"

#define VIR_FROM_THIS VIR_FROM_BHYVE

VIR_LOG_INIT("bhyve.bhyve_command");

static int
bhyveBuildNetArgStr(const virDomainDef *def,
                    virDomainNetDef *net,
                    struct _bhyveConn *driver,
                    virCommand *cmd,
                    bool dryRun)
{
    char macaddr[VIR_MAC_STRING_BUFLEN];
    char *realifname = NULL;
    const char *brname = NULL;
    const char *nic_model = NULL;
    int ret = -1;
    virDomainNetType actualType = virDomainNetGetActualType(net);
    g_autoptr(virConnect) netconn = NULL;

    if (net->model == VIR_DOMAIN_NET_MODEL_VIRTIO) {
        nic_model = "virtio-net";
    } else if (net->model == VIR_DOMAIN_NET_MODEL_E1000) {
        if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_NET_E1000) != 0) {
            nic_model = "e1000";
        } else {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("NIC model 'e1000' is not supported by given bhyve binary"));
            return -1;
        }
    } else {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("NIC model is not supported"));
        return -1;
    }

    if (net->type == VIR_DOMAIN_NET_TYPE_NETWORK) {
        if (!netconn && !(netconn = virGetConnectNetwork()))
            return -1;
        if (virDomainNetAllocateActualDevice(netconn, def, net) < 0)
            return -1;
    }
    /* final validation now that actual type is known */
    if (virDomainActualNetDefValidate(net) < 0)
        return -1;

    virMacAddrFormat(&net->mac, macaddr);

    switch (actualType) {
    case VIR_DOMAIN_NET_TYPE_NETWORK:
    case VIR_DOMAIN_NET_TYPE_BRIDGE:
        brname = virDomainNetGetActualBridgeName(net);
        if (!brname) {
            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                           _("No bridge name specified"));
            return -1;
        }
        break;
    case VIR_DOMAIN_NET_TYPE_USER:
        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:0,%s,slirp,mac=%s,open",
                               net->info.addr.pci.slot, nic_model, macaddr);
        return 0;
    case VIR_DOMAIN_NET_TYPE_ETHERNET:
    case VIR_DOMAIN_NET_TYPE_DIRECT:
    case VIR_DOMAIN_NET_TYPE_VHOSTUSER:
    case VIR_DOMAIN_NET_TYPE_SERVER:
    case VIR_DOMAIN_NET_TYPE_CLIENT:
    case VIR_DOMAIN_NET_TYPE_MCAST:
    case VIR_DOMAIN_NET_TYPE_UDP:
    case VIR_DOMAIN_NET_TYPE_INTERNAL:
    case VIR_DOMAIN_NET_TYPE_HOSTDEV:
    case VIR_DOMAIN_NET_TYPE_VDPA:
    case VIR_DOMAIN_NET_TYPE_NULL:
    case VIR_DOMAIN_NET_TYPE_VDS:
    case VIR_DOMAIN_NET_TYPE_LAST:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("Unsupported network type %1$s"),
                       virDomainNetTypeToString(actualType));
        return -1;
    }

    if (!dryRun) {
        if (virNetDevTapCreateInBridgePort(brname, &net->ifname, &net->mac,
                                           def->uuid, NULL, NULL, 0,
                                           virDomainNetGetActualVirtPortProfile(net),
                                           virDomainNetGetActualVlan(net),
                                           virDomainNetGetActualPortOptionsIsolated(net),
                                           NULL, 0, NULL,
                                           VIR_NETDEV_TAP_CREATE_IFUP | VIR_NETDEV_TAP_CREATE_PERSIST) < 0) {
            goto cleanup;
        }

        realifname = virNetDevTapGetRealDeviceName(net->ifname);

        if (realifname == NULL)
            goto cleanup;

        VIR_DEBUG("%s -> %s", net->ifname, realifname);
        /* hack on top of other hack: we need to set
         * interface to 'UP' again after re-opening to find its
         * name
         */
        if (virNetDevSetOnline(net->ifname, true) != 0)
            goto cleanup;
    } else {
        realifname = g_strdup("tap0");
    }


    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:0,%s,%s,mac=%s",
                           net->info.addr.pci.slot, nic_model,
                           realifname, macaddr);

    ret = 0;
 cleanup:
    if (ret < 0)
        VIR_FREE(net->ifname);
    VIR_FREE(realifname);

    return ret;
}

static int
bhyveBuildConsoleArgStr(const virDomainDef *def, virCommand *cmd)
{
    size_t i = 0;
    virDomainChrDef *chr = NULL;

    if (!def->nserials)
        return 0;

    if (ARCH_IS_X86(def->os.arch)) {
        for (i = 0; i < def->nserials; i++) {
            chr = def->serials[i];

            /* bhyve supports 4 ports: com1, com2, com3, com4 */
            if (chr->target.port > 3) {
                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                               _("Only four serial ports are supported"));
                return -1;
            }

            virCommandAddArg(cmd, "-l");

            switch (chr->source->type) {
            case VIR_DOMAIN_CHR_TYPE_NMDM:
                virCommandAddArgFormat(cmd, "com%d,%s",
                                       chr->target.port + 1, chr->source->data.file.path);
                break;
            case VIR_DOMAIN_CHR_TYPE_TCP:
                virCommandAddArgFormat(cmd, "com%d,tcp=%s:%s",
                                       chr->target.port + 1,
                                       chr->source->data.tcp.host,
                                       chr->source->data.tcp.service);
                break;
            default:
                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                               _("Only 'nmdm' and 'tcp' console types are supported"));
                return -1;
            }
        }
    } else if (ARCH_IS_ARM(def->os.arch)) {
        if (def->nserials > 1) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Only one console is supported on arm64"));
            return -1;
        }

        chr = def->serials[0];
        if (chr->source->type != VIR_DOMAIN_CHR_TYPE_NMDM) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Only 'nmdm' console type is supported on arm64"));
            return -1;
        }

        virCommandAddArg(cmd, "-o");
        virCommandAddArgFormat(cmd, "console=%s",
                               chr->source->data.file.path);
    }

    return 0;
}

static int
bhyveBuildRNGArgStr(const virDomainDef *def G_GNUC_UNUSED,
                    virDomainRNGDef *rng,
                    virCommand *cmd)
{
    if (rng->backend != VIR_DOMAIN_RNG_BACKEND_RANDOM) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("RNG backend is not supported"));
        return -1;
    }

    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:%d,virtio-rnd",
                           rng->info.addr.pci.slot,
                           rng->info.addr.pci.function);
    return 0;
}

static int
bhyveBuildHostdevArgStr(const virDomainDef *def,
                        virCommand *cmd)
{
    size_t i;

    for (i = 0; i < def->nhostdevs; i++) {
        virDomainHostdevDef *hostdev = def->hostdevs[i];
        virDomainHostdevSubsys *subsys = &hostdev->source.subsys;

        if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS ||
            subsys->type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) {
            continue;
        }

        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:%d,passthru,%d/%d/%d",
                               hostdev->info->addr.pci.slot,
                               hostdev->info->addr.pci.function,
                               subsys->u.pci.addr.bus,
                               subsys->u.pci.addr.slot,
                               subsys->u.pci.addr.function);
    }

    return 0;
}

static int
bhyveBuildAHCIControllerArgStr(const virDomainDef *def,
                               virDomainControllerDef *controller,
                               struct _bhyveConn *driver,
                               virCommand *cmd)
{
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
    const char *disk_source;
    size_t i;

    for (i = 0; i < def->ndisks; i++) {
        g_auto(virBuffer) device = VIR_BUFFER_INITIALIZER;
        virDomainDiskDef *disk = def->disks[i];

        if (disk->bus != VIR_DOMAIN_DISK_BUS_SATA)
            continue;

        if (disk->info.addr.drive.controller != controller->idx)
            continue;

        VIR_DEBUG("disk %zu controller %d", i, controller->idx);

        if ((virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_FILE) &&
            (virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_VOLUME)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("unsupported disk type"));
            return -1;
        }

        if (virDomainDiskTranslateSourcePool(disk) < 0)
            return -1;

        disk_source = virDomainDiskGetSource(disk);

        if ((disk->device == VIR_DOMAIN_DISK_DEVICE_CDROM) &&
            (disk_source == NULL)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("cdrom device without source path not supported"));
            return -1;
        }

        switch (disk->device) {
        case VIR_DOMAIN_DISK_DEVICE_DISK:
            if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_AHCI32SLOT))
                virBufferAsprintf(&device, ",hd:%s", disk_source);
            else
                virBufferAsprintf(&device, "-hd,%s", disk_source);
            break;
        case VIR_DOMAIN_DISK_DEVICE_CDROM:
            if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_AHCI32SLOT))
                virBufferAsprintf(&device, ",cd:%s", disk_source);
            else
                virBufferAsprintf(&device, "-cd,%s", disk_source);
            break;
        case VIR_DOMAIN_DISK_DEVICE_FLOPPY:
        case VIR_DOMAIN_DISK_DEVICE_LUN:
        case VIR_DOMAIN_DISK_DEVICE_LAST:
        default:
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("unsupported disk device"));
            return -1;
        }

        if (disk->rotation_rate)
            virBufferAsprintf(&device, ",nmrr=%u", disk->rotation_rate);

        virBufferAddBuffer(&buf, &device);
    }

    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:0,ahci%s",
                           controller->info.addr.pci.slot,
                           virBufferCurrentContent(&buf));

    return 0;
}

static int
bhyveBuildSCSIControllerArgStr(const virDomainDef *def,
                               virDomainControllerDef *controller,
                               struct _bhyveConn *driver G_GNUC_UNUSED,
                               virCommand *cmd)
{
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
    const char *disk_source;
    size_t i;

    for (i = 0; i < def->ndisks; i++) {
        g_auto(virBuffer) device = VIR_BUFFER_INITIALIZER;
        virDomainDiskDef *disk = def->disks[i];

        if (disk->bus != VIR_DOMAIN_DISK_BUS_SCSI)
            continue;

        if (disk->info.addr.drive.controller != controller->idx)
            continue;

        VIR_DEBUG("disk %zu controller %d", i, controller->idx);

        if (virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_CTL) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("unsupported disk type"));
            return -1;
        }

        if (virDomainDiskTranslateSourcePool(disk) < 0)
            return -1;

        disk_source = virDomainDiskGetSource(disk);

        if ((disk->device == VIR_DOMAIN_DISK_DEVICE_CDROM) &&
            (disk_source == NULL)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("cdrom device without source path not supported"));
            return -1;
        }

        if (disk->device != VIR_DOMAIN_DISK_DEVICE_DISK) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("unsupported disk device"));
            return -1;
        }

        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:0,virtio-scsi,%s",
                               controller->info.addr.pci.slot,
                               virDomainDiskGetSource(disk));

        return 0;
    }

    return 0;
}

static int
bhyveBuildUSBControllerArgStr(const virDomainDef *def,
                              virDomainControllerDef *controller,
                              virCommand *cmd)
{
    size_t i;
    int ndevices = 0;

    for (i = 0; i < def->ninputs; i++) {
        virDomainInputDef *input = def->inputs[i];

        if (input->bus != VIR_DOMAIN_INPUT_BUS_USB) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("only USB input devices are supported"));
            return -1;
        }

        if (input->type != VIR_DOMAIN_INPUT_TYPE_TABLET) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("only tablet input devices are supported"));
            return -1;
        }
        ndevices++;
    }

    if (ndevices != 1) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("only single input device is supported"));
        return -1;
    }

    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:%d,xhci,tablet",
                           controller->info.addr.pci.slot,
                           controller->info.addr.pci.function);

    return 0;
}

static int
bhyveBuildNVMeControllerArgStr(const virDomainDef *def,
                               virDomainControllerDef *controller,
                               struct _bhyveConn *driver,
                               virCommand *cmd)
{
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
    const char *disk_source;
    size_t i;

    for (i = 0; i < def->ndisks; i++) {
        g_autofree char *nvme_opts = NULL;

        g_auto(virBuffer) opt = VIR_BUFFER_INITIALIZER;
        virDomainDiskDef *disk = def->disks[i];

        if (disk->bus != VIR_DOMAIN_DISK_BUS_NVME)
            continue;

        if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_NVME)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Bhyve version does not support NVMe"));
            return -1;
        }

        if (disk->info.addr.drive.controller != controller->idx)
            continue;

        VIR_DEBUG("disk %zu controller %d", i, controller->idx);

        disk_source = virDomainDiskGetSource(disk);

        if (disk->queues)
            virBufferAsprintf(&opt, ",maxq=%d", disk->queues);
        if (disk->queue_size)
            virBufferAsprintf(&opt, ",qsz=%d", disk->queue_size);

        nvme_opts = virBufferContentAndReset(&opt);

        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:0,nvme,%s%s",
                               controller->info.addr.pci.slot,
                               disk_source,
                               NULLSTR_EMPTY(nvme_opts));

    }

    return 0;
}

static int
bhyveBuildVirtioSerialControllerArgStr(const virDomainDef *def,
                                       virDomainControllerDef *controller,
                                       struct _bhyveConn *driver G_GNUC_UNUSED,
                                       virCommand *cmd)
{
    g_auto(virBuffer) opt = VIR_BUFFER_INITIALIZER;
    size_t i;

    for (i = 0; i < def->nchannels; i++) {
        virDomainChrDef *channel = def->channels[i];

        if (channel->info.addr.vioserial.controller != controller->idx)
            continue;

        if (channel->source->type != VIR_DOMAIN_CHR_TYPE_UNIX)
            continue;

        if (channel->targetType != VIR_DOMAIN_CHR_CHANNEL_TARGET_TYPE_VIRTIO)
            continue;

        virBufferAsprintf(&opt,
                          ",%s=%s",
                          channel->target.name,
                          channel->source->data.nix.path);
    }

    if (virBufferUse(&opt) > 0) {
        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:0,virtio-console%s",
                               controller->info.addr.pci.slot,
                               virBufferContentAndReset(&opt));
    }

    return 0;
}

static int
bhyveBuildVirtIODiskArgStr(const virDomainDef *def G_GNUC_UNUSED,
                           virDomainDiskDef *disk,
                           virCommand *cmd)
{
    const char *disk_source;

    if (virDomainDiskTranslateSourcePool(disk) < 0)
        return -1;

    if (disk->device != VIR_DOMAIN_DISK_DEVICE_DISK) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported disk device"));
        return -1;
    }

    if ((virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_FILE) &&
        (virDomainDiskGetType(disk) != VIR_STORAGE_TYPE_VOLUME)) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported disk type"));
        return -1;
    }

    disk_source = virDomainDiskGetSource(disk);

    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:0,virtio-blk,%s",
                           disk->info.addr.pci.slot,
                           disk_source);

    return 0;
}

static int
bhyveBuildDiskArgStr(const virDomainDef *def,
                     virDomainDiskDef *disk,
                     virCommand *cmd)
{
    switch (disk->bus) {
    case VIR_DOMAIN_DISK_BUS_SATA:
        /* Handled by bhyveBuildAHCIControllerArgStr() */
        break;
    case VIR_DOMAIN_DISK_BUS_SCSI:
        /* Handled by bhyveBuildSCSIControllerArgStr() */
        break;
    case VIR_DOMAIN_DISK_BUS_NVME:
        /* Handled by bhyveBuildNVMeControllerArgStr() */
        break;
    case VIR_DOMAIN_DISK_BUS_VIRTIO:
        if (bhyveBuildVirtIODiskArgStr(def, disk, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_DISK_BUS_IDE:
    case VIR_DOMAIN_DISK_BUS_FDC:
    case VIR_DOMAIN_DISK_BUS_NONE:
    case VIR_DOMAIN_DISK_BUS_XEN:
    case VIR_DOMAIN_DISK_BUS_USB:
    case VIR_DOMAIN_DISK_BUS_UML:
    case VIR_DOMAIN_DISK_BUS_SD:
    case VIR_DOMAIN_DISK_BUS_LAST:
    default:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported disk device"));
        return -1;
    }
    return 0;
}

static int
bhyveBuildControllerArgStr(const virDomainDef *def,
                           virDomainControllerDef *controller,
                           struct _bhyveConn *driver,
                           virCommand *cmd,
                           unsigned *nusbcontrollers,
                           unsigned *nisacontrollers)
{
    switch (controller->type) {
    case VIR_DOMAIN_CONTROLLER_TYPE_PCI:
        if (controller->model != VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("unsupported PCI controller model: only PCI root supported"));
            return -1;
        }
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_SATA:
        if (bhyveBuildAHCIControllerArgStr(def, controller, driver, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_SCSI:
        if (bhyveBuildSCSIControllerArgStr(def, controller, driver, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_USB:
        if (++*nusbcontrollers > 1) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("only single USB controller is supported"));
            return -1;
        }

        if (bhyveBuildUSBControllerArgStr(def, controller, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_ISA:
        if (++*nisacontrollers > 1) {
             virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                            "%s", _("only single ISA controller is supported"));
             return -1;
        }
        virCommandAddArg(cmd, "-s");
        virCommandAddArgFormat(cmd, "%d:0,lpc",
                                controller->info.addr.pci.slot);
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_NVME:
        if (bhyveBuildNVMeControllerArgStr(def, controller, driver, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_VIRTIO_SERIAL:
        if (bhyveBuildVirtioSerialControllerArgStr(def, controller, driver, cmd) < 0)
            return -1;
        break;
    case VIR_DOMAIN_CONTROLLER_TYPE_IDE:
    case VIR_DOMAIN_CONTROLLER_TYPE_FDC:
    case VIR_DOMAIN_CONTROLLER_TYPE_CCID:
    case VIR_DOMAIN_CONTROLLER_TYPE_XENBUS:
    case VIR_DOMAIN_CONTROLLER_TYPE_LAST:
    default:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported controller device"));
        return -1;

    }
    return 0;
}

static int
bhyveBuildGraphicsArgStr(const virDomainDef *def,
                         virDomainGraphicsDef *graphics,
                         virDomainVideoDef *video,
                         struct _bhyveConn *driver,
                         virCommand *cmd,
                         bool dryRun)
{
    g_auto(virBuffer) opt = VIR_BUFFER_INITIALIZER;
    virDomainGraphicsListenDef *glisten = NULL;
    bool escapeAddr;
    unsigned short port;

    if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_LPC_BOOTROM) ||
        def->os.bootloader ||
        !def->os.loader) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Graphics are only supported when booting using UEFI"));
        return -1;
    }

    if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_FBUF)) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Bhyve version does not support framebuffer"));
        return -1;
    }

    if (graphics->type != VIR_DOMAIN_GRAPHICS_TYPE_VNC) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Only VNC supported"));
        return -1;
    }

    if (!(glisten = virDomainGraphicsGetListen(graphics, 0))) {
        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                       _("Missing listen element"));
        return -1;
    }

    virBufferAsprintf(&opt, "%d:%d,fbuf", video->info.addr.pci.slot, video->info.addr.pci.function);

    switch (glisten->type) {
    case VIR_DOMAIN_GRAPHICS_LISTEN_TYPE_ADDRESS:
    case VIR_DOMAIN_GRAPHICS_LISTEN_TYPE_NETWORK:
        virBufferAddLit(&opt, ",tcp=");

        if (!graphics->data.vnc.autoport &&
            (graphics->data.vnc.port < 5900 ||
             graphics->data.vnc.port > 65535)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("vnc port must be in range [5900,65535]"));
            return -1;
        }

        if (glisten->address) {
            escapeAddr = strchr(glisten->address, ':') != NULL;
            if (escapeAddr)
                virBufferAsprintf(&opt, "[%s]", glisten->address);
            else
                virBufferAdd(&opt, glisten->address, -1);
        }

        if (!dryRun) {
            if (graphics->data.vnc.autoport) {
                if (virPortAllocatorAcquire(driver->remotePorts, &port) < 0)
                    return -1;
                graphics->data.vnc.port = port;
            } else {
                if (virPortAllocatorSetUsed(graphics->data.vnc.port) < 0)
                    VIR_WARN("Failed to mark VNC port '%d' as used by '%s'",
                             graphics->data.vnc.port, def->name);
            }
        }

        virBufferAsprintf(&opt, ":%d", graphics->data.vnc.port);
        break;
    case VIR_DOMAIN_GRAPHICS_LISTEN_TYPE_SOCKET:
    case VIR_DOMAIN_GRAPHICS_LISTEN_TYPE_NONE:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Unsupported listen type"));
        return -1;
    case VIR_DOMAIN_GRAPHICS_LISTEN_TYPE_LAST:
    default:
        virReportEnumRangeError(virDomainGraphicsListenType, glisten->type);
        return -1;
    }

    if (graphics->data.vnc.auth.passwd) {
        if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_VNC_PASSWORD)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("VNC Password authentication not supported by bhyve"));
            return -1;
        }

        if (strchr(graphics->data.vnc.auth.passwd, ',')) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Password may not contain ',' character"));
            return -1;
        }

        virBufferAsprintf(&opt, ",password=%s", graphics->data.vnc.auth.passwd);
    } else {
        if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_VNC_PASSWORD))
            VIR_WARN("%s", _("Security warning: VNC auth is not supported."));
        else
            VIR_WARN("%s", _("Security warning: VNC is used without authentication."));
    }

    if (video->res)
        virBufferAsprintf(&opt, ",w=%d,h=%d", video->res->x, video->res->y);

    if (video->driver)
        virBufferAsprintf(&opt, ",vga=%s",
                          virDomainVideoVGAConfTypeToString(video->driver->vgaconf));

    if (graphics->data.vnc.wait == VIR_TRISTATE_BOOL_YES)
        virBufferAddLit(&opt, ",wait");

    virCommandAddArg(cmd, "-s");
    virCommandAddArgBuffer(cmd, &opt);
    return 0;

}

static int
bhyveBuildSoundArgStr(const virDomainDef *def G_GNUC_UNUSED,
                      virDomainSoundDef *sound,
                      virDomainAudioDef *audio,
                      struct _bhyveConn *driver,
                      virCommand *cmd)
{
    g_auto(virBuffer) params = VIR_BUFFER_INITIALIZER;

    if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_SOUND_HDA)) {
        /* Currently, bhyve only supports "hda" sound devices, so
           if it's not supported, sound devices are not supported at all */
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Sound devices emulation is not supported by given bhyve binary"));
        return -1;
    }

    if (sound->model != VIR_DOMAIN_SOUND_MODEL_ICH7) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Sound device model is not supported"));
        return -1;
    }

    virCommandAddArg(cmd, "-s");

    if (audio) {
        switch (audio->type) {
        case  VIR_DOMAIN_AUDIO_TYPE_OSS:
            if (virDomainAudioIOCommonIsSet(&audio->input) ||
                virDomainAudioIOCommonIsSet(&audio->output)) {
                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                               _("cannot set common audio backend settings"));
                return -1;
            }

            if (audio->backend.oss.input.dev)
                virBufferAsprintf(&params, ",play=%s",
                                  audio->backend.oss.input.dev);

            if (audio->backend.oss.output.dev)
                virBufferAsprintf(&params, ",rec=%s",
                                  audio->backend.oss.output.dev);

            break;

        case VIR_DOMAIN_AUDIO_TYPE_NONE:
        case VIR_DOMAIN_AUDIO_TYPE_ALSA:
        case VIR_DOMAIN_AUDIO_TYPE_COREAUDIO:
        case VIR_DOMAIN_AUDIO_TYPE_JACK:
        case VIR_DOMAIN_AUDIO_TYPE_PULSEAUDIO:
        case VIR_DOMAIN_AUDIO_TYPE_SDL:
        case VIR_DOMAIN_AUDIO_TYPE_SPICE:
        case VIR_DOMAIN_AUDIO_TYPE_FILE:
        case VIR_DOMAIN_AUDIO_TYPE_DBUS:
        case VIR_DOMAIN_AUDIO_TYPE_PIPEWIRE:
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                           _("unsupported audio backend '%1$s'"),
                           virDomainAudioTypeTypeToString(audio->type));
            return -1;
        case VIR_DOMAIN_AUDIO_TYPE_LAST:
        default:
            virReportEnumRangeError(virDomainAudioType, audio->type);
            return -1;
        }
    }

    virCommandAddArgFormat(cmd, "%d:%d,hda%s",
                           sound->info.addr.pci.slot,
                           sound->info.addr.pci.function,
                           virBufferCurrentContent(&params));

    return 0;
}

static int
bhyveBuildFSArgStr(const virDomainDef *def G_GNUC_UNUSED,
                   virDomainFSDef *fs,
                   virCommand *cmd)
{
    g_auto(virBuffer) params = VIR_BUFFER_INITIALIZER;

    switch (fs->type) {
    case VIR_DOMAIN_FS_TYPE_MOUNT:
        break;
    case VIR_DOMAIN_FS_TYPE_BLOCK:
    case VIR_DOMAIN_FS_TYPE_FILE:
    case VIR_DOMAIN_FS_TYPE_TEMPLATE:
    case VIR_DOMAIN_FS_TYPE_RAM:
    case VIR_DOMAIN_FS_TYPE_BIND:
    case VIR_DOMAIN_FS_TYPE_VOLUME:
    case VIR_DOMAIN_FS_TYPE_LAST:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("unsupported filesystem type '%1$s'"),
                       virDomainFSTypeToString(fs->type));
        return -1;
    }

    switch (fs->fsdriver) {
    case VIR_DOMAIN_FS_DRIVER_TYPE_DEFAULT:
        /* The only supported driver by bhyve currently */
        break;
    case VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS:
    case VIR_DOMAIN_FS_DRIVER_TYPE_PATH:
    case VIR_DOMAIN_FS_DRIVER_TYPE_HANDLE:
    case VIR_DOMAIN_FS_DRIVER_TYPE_LOOP:
    case VIR_DOMAIN_FS_DRIVER_TYPE_NBD:
    case VIR_DOMAIN_FS_DRIVER_TYPE_PLOOP:
    case VIR_DOMAIN_FS_DRIVER_TYPE_MTP:
    case VIR_DOMAIN_FS_DRIVER_TYPE_LAST:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("unsupported filesystem driver '%1$s'"),
                       virDomainFSDriverTypeToString(fs->fsdriver));
        return -1;
    }

    switch (fs->accessmode) {
    case VIR_DOMAIN_FS_ACCESSMODE_PASSTHROUGH:
        /* This is the only supported mode for now, does not need specific configuration */
        break;
    case VIR_DOMAIN_FS_ACCESSMODE_MAPPED:
    case VIR_DOMAIN_FS_ACCESSMODE_SQUASH:
    case VIR_DOMAIN_FS_ACCESSMODE_DEFAULT:
    case VIR_DOMAIN_FS_ACCESSMODE_LAST:
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("unsupported filesystem accessmode '%1$s'"),
                       virDomainFSAccessModeTypeToString(fs->accessmode));
        return -1;
    }

    if (fs->readonly)
        virBufferAddLit(&params, ",ro");

    virCommandAddArg(cmd, "-s");
    virCommandAddArgFormat(cmd, "%d:%d,virtio-9p,%s=%s%s",
                           fs->info.addr.pci.slot,
                           fs->info.addr.pci.function,
                           fs->dst,
                           fs->src->path,
                           virBufferCurrentContent(&params));

    return 0;
}

static void
bhyveBuildSysinfoBiosArgStr(virSysinfoBIOSDef *def,
                            virCommand *cmd)
{
    if (def->vendor) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "bios.vendor", def->vendor);
    }

    if (def->version) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "bios.version", def->version);
    }

    if (def->date) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "bios.release_date", def->date);
    }
}

static void
bhyveBuildSysinfoSystemArgStr(virSysinfoSystemDef *def,
                              virCommand *cmd)
{
    if (def->manufacturer) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "system.manufacturer", def->manufacturer);
    }

    if (def->product) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "system.product_name", def->product);
    }

    if (def->serial) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "system.serial_number", def->serial);
    }

    if (def->sku) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "system.sku", def->sku);
    }

    if (def->version) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "system.version", def->version);
    }
}

static void
bhyveBuildSysinfoBaseBoardArgStr(virSysinfoBaseBoardDef *def,
                                 virCommand *cmd)
{
    if (def->manufacturer) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.manufacturer", def->manufacturer);
    }

    if (def->product) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.product_name", def->product);
    }

    if (def->version) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.version", def->version);
    }

    if (def->serial) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.serial_number", def->serial);
    }

    if (def->asset) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.asset_tag", def->asset);
    }

    if (def->location) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "board.location", def->location);
    }
}

static void
bhyveBuildSysinfoChassisArgStr(virSysinfoChassisDef *def,
                               virCommand *cmd)
{
    if (def->manufacturer) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "chassis.manufacturer", def->manufacturer);
    }

    if (def->version) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "chassis.version", def->version);
    }

    if (def->serial) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "chassis.serial_number", def->serial);
    }

    if (def->asset) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "chassis.asset_tag", def->asset);
    }

    if (def->sku) {
        virCommandAddArg(cmd, "-o");
        virCommandAddArgPair(cmd, "chassis.sku", def->sku);
    }
}

static int
bhyveBuildSysinfoArgStr(virSysinfoDef *sysinfo,
                        virCommand *cmd)
{
    size_t i;

    if (sysinfo->bios)
        bhyveBuildSysinfoBiosArgStr(sysinfo->bios, cmd);

    if (sysinfo->system)
        bhyveBuildSysinfoSystemArgStr(sysinfo->system, cmd);

    if (sysinfo->nbaseBoard > 1) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("bhyve does not support more than one entry to Type 2 in SMBIOS table"));
        return -1;
    }

    for (i = 0; i < sysinfo->nbaseBoard; i++)
        bhyveBuildSysinfoBaseBoardArgStr(&sysinfo->baseBoard[i], cmd);

    if (sysinfo->chassis)
        bhyveBuildSysinfoChassisArgStr(sysinfo->chassis, cmd);

    return 0;
}

virCommand *
virBhyveProcessBuildBhyveCmd(struct _bhyveConn *driver, virDomainDef *def,
                             bool dryRun)
{
    /*
     * /usr/sbin/bhyve -c 2 -m 256 -AI -H -P \
     *            -s 0:0,hostbridge \
     *            -s 1:0,virtio-net,tap0 \
     *            -s 2:0,ahci-hd,${IMG} \
     *            -S 31,uart,stdio \
     *            vm0
     */
    g_autoptr(virCommand) cmd = virCommandNew("bhyve");
    size_t i;
    unsigned nusbcontrollers = 0;
    unsigned nisacontrollers = 0;
    unsigned nvcpus = virDomainDefGetVcpus(def);
    size_t ncells = virDomainNumaGetNodeCount(def->numa);

    /* CPUs */
    virCommandAddArg(cmd, "-c");
    if (def->cpu && def->cpu->sockets) {
        if (def->cpu->dies != 1) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Only 1 die per socket is supported"));
            return NULL;
        }
        if (def->cpu->clusters != 1) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Only 1 cluster per die is supported"));
            return NULL;
        }
        if (nvcpus != def->cpu->sockets * def->cpu->cores * def->cpu->threads) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Invalid CPU topology: total number of vCPUs must equal the product of sockets, cores, and threads"));
            return NULL;
        }

        if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_CPUTOPOLOGY) != 0) {
            virCommandAddArgFormat(cmd, "cpus=%d,sockets=%d,cores=%d,threads=%d",
                                   nvcpus,
                                   def->cpu->sockets,
                                   def->cpu->cores,
                                   def->cpu->threads);
        } else {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Installed bhyve binary does not support defining CPU topology"));
            return NULL;
        }
    } else {
        virCommandAddArgFormat(cmd, "%d", nvcpus);
    }

    /* CPU tuning */
    for (i = 0; i < virDomainDefGetVcpusMax(def); i++) {
        virDomainVcpuDef *vcpu = virDomainDefGetVcpu(def, i);

        if (vcpu->cpumask) {
            ssize_t j = -1;

            while ((j = virBitmapNextSetBit(vcpu->cpumask, j)) >= 0) {
                virCommandAddArg(cmd, "-p");
                virCommandAddArgFormat(cmd, "%zu:%zu", i, j);
            }

        }
    }

    /* NUMA */
    if (ncells) {
        if (!(bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_NUMA)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Installed bhyve binary does not support NUMA configuration"));
            return NULL;
        }

        if (def->os.bootloader || !def->os.loader) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("NUMA configuration is only supported when booting using UEFI"));
            return NULL;
        }

        for (i = 0; i < ncells; i++) {
            unsigned long long memSize = virDomainNumaGetNodeMemorySize(def->numa, i);
            virBitmap *cpus = virDomainNumaGetNodeCpumask(def->numa, i);
            g_autofree char *cpumask = virBitmapFormat(cpus);

            virCommandAddArg(cmd, "-n");
            virCommandAddArgFormat(cmd, "id=%zu,size=%llu,cpus=%s", i, VIR_DIV_UP(memSize, 1024),
                                   cpumask);
        }
    }

    /* Memory */
    virCommandAddArg(cmd, "-m");
    virCommandAddArgFormat(cmd, "%llu",
                           VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024));

    if (def->mem.locked)
        virCommandAddArg(cmd, "-S"); /* Wire guest memory */

    /* Options */
    if ((def->features[VIR_DOMAIN_FEATURE_ACPI] == VIR_TRISTATE_SWITCH_ON) &&
        (bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_ACPI))
        /* As of FreeBSD commit
         * https://cgit.freebsd.org/src/commit/?id=6a0e7f908802b86ca5d1c0b3c404b8391d0f626e
         * bhyve(8) generates ACPI tables unconditionally, so nothing needs to be done
         * if the capability is missing. */
        virCommandAddArg(cmd, "-A"); /* Create an ACPI table */
    if (def->features[VIR_DOMAIN_FEATURE_MSRS] == VIR_TRISTATE_SWITCH_ON) {
        if (def->msrs_features[VIR_DOMAIN_MSRS_UNKNOWN] == VIR_DOMAIN_MSRS_UNKNOWN_IGNORE)
            virCommandAddArg(cmd, "-w");
    }

    switch (def->clock.offset) {
    case VIR_DOMAIN_CLOCK_OFFSET_LOCALTIME:
        /* used by default in bhyve */
        break;
    case VIR_DOMAIN_CLOCK_OFFSET_UTC:
        if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_RTC_UTC) != 0) {
            virCommandAddArg(cmd, "-u");
        } else {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Installed bhyve binary does not support UTC clock"));
            return NULL;
        }
        break;
    default:
         virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                        _("unsupported clock offset '%1$s'"),
                        virDomainClockOffsetTypeToString(def->clock.offset));
         return NULL;
    }

    /* Clarification about -H and -P flags from Peter Grehan:
     * -H and -P flags force the guest to exit when it executes IA32 HLT and PAUSE
     * instructions respectively.
     *
     * For the HLT exit, bhyve uses that to infer that the guest is idling and can
     * be put to sleep until an external event arrives. If this option is not used,
     * the guest will always use 100% of CPU on the host.
     *
     * The PAUSE exit is most useful when there are large numbers of guest VMs running,
     * since it forces the guest to exit when it spins on a lock acquisition.
     *
     * These flags are currently supported on amd64 only.
     */
    if (ARCH_IS_X86(def->os.arch)) {
        virCommandAddArg(cmd, "-H"); /* vmexit from guest on hlt */
        virCommandAddArg(cmd, "-P"); /* vmexit from guest on pause */
    }

    virCommandAddArgList(cmd, "-s", "0:0,hostbridge", NULL);

    if (def->os.bootloader == NULL &&
        def->os.loader) {
        virArch arch = def->os.arch;
        g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;

        if (ARCH_IS_X86(arch)) {
            if ((bhyveDriverGetBhyveCaps(driver) & BHYVE_CAP_LPC_BOOTROM)) {
                virBufferAsprintf(&buf, "bootrom,%s", def->os.loader->path);
                if (def->os.loader->nvram && def->os.loader->nvram->path)
                    virBufferAsprintf(&buf, ",%s", def->os.loader->nvram->path);

                virCommandAddArgList(cmd, "-l", virBufferCurrentContent(&buf), NULL);
            } else {
                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                               _("Installed bhyve binary does not support UEFI loader"));
                return NULL;
            }
        } else if (ARCH_IS_ARM(arch)) {
            virBufferAsprintf(&buf, "bootrom=%s", def->os.loader->path);
            virCommandAddArgList(cmd, "-o", virBufferCurrentContent(&buf), NULL);
        }
    }

    /* Devices */
    for (i = 0; i < def->ncontrollers; i++) {
        if (bhyveBuildControllerArgStr(def, def->controllers[i], driver, cmd,
                                       &nusbcontrollers, &nisacontrollers) < 0)
            return NULL;
    }
    for (i = 0; i < def->nnets; i++) {
        if (bhyveBuildNetArgStr(def, def->nets[i], driver, cmd, dryRun) < 0)
            return NULL;
    }
    for (i = 0; i < def->ndisks; i++) {
        if (bhyveBuildDiskArgStr(def, def->disks[i], cmd) < 0)
            return NULL;
    }

    if (def->ngraphics && def->nvideos) {
        if (def->ngraphics == 1 && def->nvideos == 1) {
            if (bhyveBuildGraphicsArgStr(def, def->graphics[0], def->videos[0],
                                         driver, cmd, dryRun) < 0)
                return NULL;
        } else {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Multiple graphics devices are not supported"));
             return NULL;
        }
    }

    for (i = 0; i < def->nsounds; i++) {
        if (bhyveBuildSoundArgStr(def, def->sounds[i],
                                  virDomainDefFindAudioByID(def, def->sounds[i]->audioId),
                                  driver, cmd) < 0)
            return NULL;
    }

    for (i = 0; i < def->nfss; i++) {
        if (bhyveBuildFSArgStr(def, def->fss[i], cmd) < 0)
            return NULL;
    }

    if (bhyveBuildConsoleArgStr(def, cmd) < 0)
        return NULL;

    for (i = 0; i < def->nrngs; i++)
        if (bhyveBuildRNGArgStr(def, def->rngs[i], cmd) < 0)
            return NULL;

    if (def->namespaceData) {
        bhyveDomainCmdlineDef *bhyvecmd;

        VIR_WARN("Booting the guest using command line pass-through feature, which could potentially cause inconsistent state and upgrade issues");

        bhyvecmd = def->namespaceData;
        for (i = 0; i < bhyvecmd->num_args; i++)
            virCommandAddArg(cmd, bhyvecmd->args[i]);
    }

    if (bhyveBuildHostdevArgStr(def, cmd) < 0)
        return NULL;

    for (i = 0; i < def->nsysinfo; i++)
        if (def->sysinfo[i]->type == VIR_SYSINFO_SMBIOS)
            if (bhyveBuildSysinfoArgStr(def->sysinfo[i], cmd) < 0)
                return NULL;

    virCommandAddArg(cmd, def->name);

    return g_steal_pointer(&cmd);
}

virCommand *
virBhyveProcessBuildDestroyCmd(struct _bhyveConn *driver G_GNUC_UNUSED,
                               virDomainDef *def)
{
    virCommand *cmd = virCommandNew("bhyvectl");

    virCommandAddArg(cmd, "--destroy");
    virCommandAddArgPair(cmd, "--vm", def->name);

    return cmd;
}

static void
virAppendBootloaderArgs(virCommand *cmd, virDomainDef *def)
{
    g_auto(GStrv) blargs = NULL;

    /* XXX: Handle quoted? */
    blargs = g_strsplit(def->os.bootloaderArgs, " ", 0);
    virCommandAddArgSet(cmd, (const char * const *)blargs);
}

static virCommand *
virBhyveProcessBuildBhyveloadCmd(virDomainDef *def,
                                 struct _bhyveConn *driver,
                                 virDomainDiskDef *disk)
{
    virCommand *cmd;
    g_autoptr(virBhyveDriverConfig) cfg = virBhyveDriverGetConfig(driver);

    if (cfg->bhyveloadTimeout > 0) {
        /* TODO: update bhyve_process.c to interpret timeout(1) exit
         * codes 124-127 to produce more meaningful error messages */
        cmd = virCommandNew("timeout");
        virCommandAddArg(cmd, "--foreground");
        virCommandAddArg(cmd, "--verbose");
        if (cfg->bhyveloadTimeoutKill > 0) {
            virCommandAddArg(cmd, "-k");
            virCommandAddArgFormat(cmd, "%ds", cfg->bhyveloadTimeoutKill);
        }
        virCommandAddArgFormat(cmd, "%ds", cfg->bhyveloadTimeout);
        virCommandAddArg(cmd, "bhyveload");
    } else {
        cmd = virCommandNew("bhyveload");
    }

    if (def->os.bootloaderArgs == NULL) {
        VIR_DEBUG("bhyveload with default arguments");

        /* Memory (MB) */
        virCommandAddArg(cmd, "-m");
        virCommandAddArgFormat(cmd, "%llu",
                               VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024));

        /* Image path */
        virCommandAddArg(cmd, "-d");
        virCommandAddArg(cmd, virDomainDiskGetSource(disk));

        /* VM name */
        virCommandAddArg(cmd, def->name);
    } else {
        VIR_DEBUG("bhyveload with arguments");
        virAppendBootloaderArgs(cmd, def);
    }

    return cmd;
}

static virCommand *
virBhyveProcessBuildCustomLoaderCmd(virDomainDef *def)
{
    virCommand *cmd;

    if (def->os.bootloaderArgs == NULL) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                       _("Custom loader requires explicit %1$s configuration"),
                       "bootloader_args");
        return NULL;
    }

    VIR_DEBUG("custom loader '%s' with arguments", def->os.bootloader);

    cmd = virCommandNew(def->os.bootloader);
    virAppendBootloaderArgs(cmd, def);
    return cmd;
}

static bool
virBhyveUsableDisk(virDomainDiskDef *disk)
{
    virStorageType disk_type = virDomainDiskGetType(disk);

    if (virDomainDiskTranslateSourcePool(disk) < 0)
        return false;

    if ((disk->device != VIR_DOMAIN_DISK_DEVICE_DISK) &&
        (disk->device != VIR_DOMAIN_DISK_DEVICE_CDROM)) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported disk device"));
        return false;
    }

    if ((disk_type != VIR_STORAGE_TYPE_FILE) &&
        (disk_type != VIR_STORAGE_TYPE_VOLUME) &&
        (disk_type != VIR_STORAGE_TYPE_CTL)) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("unsupported disk type"));
        return false;
    }

    return true;
}

static void
virBhyveFormatGrubDevice(virBuffer *devicemap, virDomainDiskDef *def)
{
    if (def->device == VIR_DOMAIN_DISK_DEVICE_CDROM)
        virBufferAsprintf(devicemap, "(cd) %s\n",
                          virDomainDiskGetSource(def));
    else
        virBufferAsprintf(devicemap, "(hd0) %s\n",
                          virDomainDiskGetSource(def));
}

static virCommand *
virBhyveProcessBuildGrubbhyveCmd(virDomainDef *def,
                                 struct _bhyveConn *driver,
                                 const char *devmap_file,
                                 char **devicesmap_out)
{
    virDomainDiskDef *hdd;
    virDomainDiskDef *cd;
    virDomainDiskDef *userdef;
    virDomainDiskDef *diskdef;
    virCommand *cmd;
    unsigned int best_idx = UINT_MAX;
    size_t i;

    if (def->os.bootloaderArgs != NULL)
        return virBhyveProcessBuildCustomLoaderCmd(def);

    /* Search disk list for CD or HDD device. We'll respect <boot order=''> if
     * present and otherwise pick the first CD or failing that HDD we come
     * across. */
    cd = hdd = userdef = NULL;
    for (i = 0; i < def->ndisks; i++) {
        if (!virBhyveUsableDisk(def->disks[i]))
            continue;

        diskdef = def->disks[i];

        if (diskdef->info.bootIndex && diskdef->info.bootIndex < best_idx) {
            userdef = diskdef;
            best_idx = userdef->info.bootIndex;
            continue;
        }

        if (cd == NULL &&
            def->disks[i]->device == VIR_DOMAIN_DISK_DEVICE_CDROM) {
            cd = diskdef;
            VIR_INFO("Picking %s as CD", virDomainDiskGetSource(cd));
        }

        if (hdd == NULL &&
            def->disks[i]->device == VIR_DOMAIN_DISK_DEVICE_DISK) {
            hdd = diskdef;
            VIR_INFO("Picking %s as HDD", virDomainDiskGetSource(hdd));
        }
    }

    cmd = virCommandNew(def->os.bootloader);

    VIR_DEBUG("grub-bhyve with default arguments");

    if (devicesmap_out != NULL) {
        g_auto(virBuffer) devicemap = VIR_BUFFER_INITIALIZER;

        /* Grub device.map (just for boot) */
        if (userdef != NULL) {
            virBhyveFormatGrubDevice(&devicemap, userdef);
        } else {
            if (hdd != NULL)
                virBhyveFormatGrubDevice(&devicemap, hdd);

            if (cd != NULL)
                virBhyveFormatGrubDevice(&devicemap, cd);
        }

        *devicesmap_out = virBufferContentAndReset(&devicemap);
    }

    virCommandAddArg(cmd, "--root");
    if (userdef != NULL) {
        if (userdef->device == VIR_DOMAIN_DISK_DEVICE_CDROM)
            virCommandAddArg(cmd, "cd");
        else
            virCommandAddArg(cmd, "hd0,msdos1");
    } else if (cd != NULL) {
        virCommandAddArg(cmd, "cd");
    } else {
        virCommandAddArg(cmd, "hd0,msdos1");
    }

    virCommandAddArg(cmd, "--device-map");
    virCommandAddArg(cmd, devmap_file);

    /* Memory in MB */
    virCommandAddArg(cmd, "--memory");
    virCommandAddArgFormat(cmd, "%llu",
                           VIR_DIV_UP(virDomainDefGetMemoryInitial(def), 1024));

    if ((bhyveDriverGetGrubCaps(driver) & BHYVE_GRUB_CAP_CONSDEV) != 0 &&
        def->nserials > 0) {
        virDomainChrDef *chr;

        chr = def->serials[0];

        if (chr->source->type != VIR_DOMAIN_CHR_TYPE_NMDM) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("only nmdm console types are supported"));
            return NULL;
        }

        virCommandAddArg(cmd, "--cons-dev");
        virCommandAddArg(cmd, chr->source->data.file.path);
    }

    /* VM name */
    virCommandAddArg(cmd, def->name);

    return cmd;
}

static virDomainDiskDef *
virBhyveGetBootDisk(virDomainDef *def)
{
    size_t i;
    virDomainDiskDef *match = NULL;
    int boot_dev = -1;

    if (def->ndisks < 1) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Domain should have at least one disk defined"));
        return NULL;
    }

    if (def->os.nBootDevs > 1) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Only one boot device is supported"));
        return NULL;
    } else if (def->os.nBootDevs == 1) {
        switch (def->os.bootDevs[0]) {
        case VIR_DOMAIN_BOOT_CDROM:
            boot_dev = VIR_DOMAIN_DISK_DEVICE_CDROM;
            break;
        case VIR_DOMAIN_BOOT_DISK:
            boot_dev = VIR_DOMAIN_DISK_DEVICE_DISK;
            break;
        case VIR_DOMAIN_BOOT_FLOPPY:
        case VIR_DOMAIN_BOOT_NET:
        case VIR_DOMAIN_BOOT_LAST:
        default:
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                           _("Cannot boot from device %1$s"),
                           virDomainBootTypeToString(def->os.bootDevs[0]));
            return NULL;
        }
    }

    if (boot_dev != -1) {
        /* If boot_dev is set, we return the first device of
         * the request type */
        for (i = 0; i < def->ndisks; i++) {
            if (!virBhyveUsableDisk(def->disks[i]))
                continue;

            if (def->disks[i]->device == boot_dev) {
                match = def->disks[i];
                break;
            }
        }

        if (match == NULL) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
                           _("Cannot find boot device of requested type %1$s"),
                           virDomainBootTypeToString(def->os.bootDevs[0]));
            return NULL;
        }
    } else {
        /* Otherwise, if boot_dev is not set, we try to find if bootIndex
         * is set for individual device. However, as bhyve does not support
         * specifying real boot priority for devices, we allow only single
         * device with boot priority set.
         */
        int first_usable_disk_index = -1;

        for (i = 0; i < def->ndisks; i++) {
            if (!virBhyveUsableDisk(def->disks[i]))
                continue;
            else
                first_usable_disk_index = i;

            if (def->disks[i]->info.bootIndex > 0) {
                if (match == NULL) {
                    match = def->disks[i];
                } else {
                    virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                                   _("Only one boot device is supported"));
                    return NULL;
                }
            }
        }

        /* If user didn't explicitly specify boot priority,
         * just return the first usable disk */
        if ((match == NULL) && (first_usable_disk_index >= 0))
            return def->disks[first_usable_disk_index];
    }

    return match;
}

virCommand *
virBhyveProcessBuildLoadCmd(struct _bhyveConn *driver, virDomainDef *def,
                            const char *devmap_file, char **devicesmap_out)
{
    virDomainDiskDef *disk = NULL;

    if (def->os.bootloader == NULL) {
        disk = virBhyveGetBootDisk(def);

        if (disk == NULL)
            return NULL;

        return virBhyveProcessBuildBhyveloadCmd(def, driver, disk);
    } else if (strstr(def->os.bootloader, "grub-bhyve") != NULL) {
        return virBhyveProcessBuildGrubbhyveCmd(def, driver, devmap_file,
                                                devicesmap_out);
    } else {
        return virBhyveProcessBuildCustomLoaderCmd(def);
    }
}
