/*
 * bhyve_driver.c: core driver methods for managing bhyve guests
 *
 * Copyright (C) 2014 Roman Bogorodskiy
 * Copyright (C) 2014-2015 Red Hat, Inc.
 * Copyright (C) 2025 The FreeBSD Foundation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include <config.h>

#include <fcntl.h>
#include <sys/utsname.h>
#include <sys/param.h>
#include <sys/sysctl.h>
#include <sys/user.h>

#include "virerror.h"
#include "datatypes.h"
#include "virbuffer.h"
#include "viruuid.h"
#include "configmake.h"
#include "viralloc.h"
#include "network_conf.h"
#include "interface_conf.h"
#include "domain_audit.h"
#include "domain_event.h"
#include "snapshot_conf.h"
#include "virfdstream.h"
#include "storage_conf.h"
#include "node_device_conf.h"
#include "virdomainobjlist.h"
#include "virxml.h"
#include "virthread.h"
#include "virlog.h"
#include "virfile.h"
#include "virpidfile.h"
#include "virtypedparam.h"
#include "virrandom.h"
#include "virstring.h"
#include "cpu/cpu.h"
#include "viraccessapicheck.h"
#include "viraccessapicheckqemu.h"
#include "virhostcpu.h"
#include "virhostmem.h"
#include "virportallocator.h"
#include "conf/domain_capabilities.h"
#include "virutil.h"
#include "domain_driver.h"
#include "virnetdevtap.h"

#include "bhyve_conf.h"
#include "bhyve_device.h"
#include "bhyve_driver.h"
#include "bhyve_command.h"
#include "bhyve_parse_command.h"
#include "bhyve_domain.h"
#include "bhyve_process.h"
#include "bhyve_capabilities.h"
#include "bhyve_rctl.h"

#define VIR_FROM_THIS   VIR_FROM_BHYVE

VIR_LOG_INIT("bhyve.bhyve_driver");

struct _bhyveConn *bhyve_driver = NULL;

static void
bhyveAutostartDomain(virDomainObj *vm, void *opaque)
{
    bhyveConn *driver = opaque;
    int ret = 0;

    ret = virBhyveProcessStart(driver, NULL, vm,
                               VIR_DOMAIN_RUNNING_BOOTED, 0);
    if (ret < 0) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       _("Failed to autostart VM '%1$s': %2$s"),
                       vm->def->name, virGetLastErrorMessage());
    }
}


static qemuAgent *
bhyveDomainObjEnterAgent(virDomainObj *obj)
{
    bhyveDomainObjPrivate *priv = obj->privateData;
    qemuAgent *agent = priv->agent;

    VIR_DEBUG("Entering agent (agent=%p vm=%p name=%s)",
              priv->agent, obj, obj->def->name);

    virObjectLock(agent);
    virObjectRef(agent);
    virObjectUnlock(obj);

    return agent;
}


static void
bhyveDomainObjExitAgent(virDomainObj *obj, qemuAgent *agent)
{
    virObjectUnlock(agent);
    virObjectUnref(agent);
    virObjectLock(obj);

    VIR_DEBUG("Exited agent (agent=%p vm=%p name=%s)",
              agent, obj, obj->def->name);
}


static int
bhyveDomainEnsureAgent(virDomainObj *vm,
                       bool reportError)
{
    bhyveDomainObjPrivate *priv = vm->privateData;

    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) {
        if (reportError) {
            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                           _("domain is not running"));
        }
        return -1;
    }

    if (priv->agent)
        return 0;

    if (!priv->eventThread &&
        virBhyveDomainObjStartWorker(vm) < 0)
        return -1;

    if (bhyveConnectAgent(NULL, vm) < 0)
        return -1;

    return 0;
}


/**
 * bhyveDriverGetCapabilities:
 *
 * Get a reference to the virCaps *instance for the
 * driver.
 *
 * The caller must release the reference with virObjetUnref
 *
 * Returns: a reference to a virCaps *instance or NULL
 */
virCaps *ATTRIBUTE_NONNULL(1)
bhyveDriverGetCapabilities(struct _bhyveConn *driver)
{
    return virObjectRef(driver->caps);
}

static char *
bhyveConnectGetCapabilities(virConnectPtr conn)
{
    struct _bhyveConn *privconn = conn->privateData;
    g_autoptr(virCaps) caps = NULL;

    if (virConnectGetCapabilitiesEnsureACL(conn) < 0)
        return NULL;

    if (!(caps = bhyveDriverGetCapabilities(privconn))) {
        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                       _("Unable to get Capabilities"));
        return NULL;
    }

    return virCapabilitiesFormatXML(caps);
}

static virDomainObj *
bhyveDomObjFromDomain(virDomainPtr domain)
{
    virDomainObj *vm;
    struct _bhyveConn *privconn = domain->conn->privateData;
    char uuidstr[VIR_UUID_STRING_BUFLEN];

    vm = virDomainObjListFindByUUID(privconn->domains, domain->uuid);
    if (!vm) {
        virUUIDFormat(domain->uuid, uuidstr);
        virReportError(VIR_ERR_NO_DOMAIN,
                       _("no domain with matching uuid '%1$s' (%2$s)"),
                       uuidstr, domain->name);
        return NULL;
    }

    return vm;
}


static int
bhyveConnectURIProbe(char **uri)
{
    if (bhyve_driver == NULL)
        return 0;

    *uri = g_strdup("bhyve:///system");
    return 1;
}


static virDrvOpenStatus
bhyveConnectOpen(virConnectPtr conn,
                 virConnectAuthPtr auth G_GNUC_UNUSED,
                 virConf *conf G_GNUC_UNUSED,
                 unsigned int flags)
{
    virCheckFlags(VIR_CONNECT_RO, VIR_DRV_OPEN_ERROR);

    if (STRNEQ(conn->uri->path, "/system")) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       _("Unexpected bhyve URI path '%1$s', try bhyve:///system"),
                       conn->uri->path);
        return VIR_DRV_OPEN_ERROR;
    }

    if (bhyve_driver == NULL) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       "%s", _("bhyve state driver is not active"));
        return VIR_DRV_OPEN_ERROR;
    }

    if (virConnectOpenEnsureACL(conn) < 0)
        return VIR_DRV_OPEN_ERROR;

    conn->privateData = bhyve_driver;

    return VIR_DRV_OPEN_SUCCESS;
}

static int
bhyveConnectClose(virConnectPtr conn)
{
    struct _bhyveConn *privconn = conn->privateData;

    virCloseCallbacksDomainRunForConn(privconn->domains, conn);
    conn->privateData = NULL;

    return 0;
}

static char *
bhyveConnectGetHostname(virConnectPtr conn)
{
    if (virConnectGetHostnameEnsureACL(conn) < 0)
        return NULL;

    return virGetHostname();
}

static char *
bhyveConnectGetSysinfo(virConnectPtr conn, unsigned int flags)
{
    struct _bhyveConn *privconn = conn->privateData;
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;

    virCheckFlags(0, NULL);

    if (virConnectGetSysinfoEnsureACL(conn) < 0)
        return NULL;

    if (!privconn->hostsysinfo) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Host SMBIOS information is not available"));
        return NULL;
    }

    if (virSysinfoFormat(&buf, privconn->hostsysinfo) < 0)
        return NULL;

    return virBufferContentAndReset(&buf);
}

static int
bhyveConnectGetVersion(virConnectPtr conn, unsigned long *version)
{
    unsigned long long tmpver;
    struct utsname ver;

    if (virConnectGetVersionEnsureACL(conn) < 0)
        return -1;

    uname(&ver);

    if (virStringParseVersion(&tmpver, ver.release, true) < 0) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       _("Unknown release: %1$s"), ver.release);
        return -1;
    }

    *version = tmpver;

    return 0;
}

static int
bhyveDomainGetInfo(virDomainPtr domain, virDomainInfoPtr info)
{
    virDomainObj *vm;
    int ret = -1;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainGetInfoEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjIsActive(vm)) {
        if (virBhyveGetDomainTotalCpuStats(vm, &(info->cpuTime)) < 0)
            goto cleanup;
    } else {
        info->cpuTime = 0;
    }

    info->state = virDomainObjGetState(vm, NULL);
    info->maxMem = virDomainDefGetMemoryTotal(vm->def);
    info->nrVirtCpu = virDomainDefGetVcpus(vm->def);
    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetState(virDomainPtr domain,
                    int *state,
                    int *reason,
                    unsigned int flags)
{
    virDomainObj *vm;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainGetStateEnsureACL(domain->conn, vm->def) < 0)
       goto cleanup;

    *state = virDomainObjGetState(vm, reason);
    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetAutostart(virDomainPtr domain, int *autostart)
{
    virDomainObj *vm;
    int ret = -1;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainGetAutostartEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    *autostart = vm->autostart;
    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainSetAutostart(virDomainPtr domain, int autostart)
{
    virDomainObj *vm;
    char *configFile = NULL;
    char *autostartLink = NULL;
    int ret = -1;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainSetAutostartEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (!vm->persistent) {
        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                       _("cannot set autostart for transient domain"));
        goto cleanup;
    }

    autostart = (autostart != 0);

    if (vm->autostart != autostart) {
        configFile = virDomainConfigFile(BHYVE_CONFIG_DIR, vm->def->name);
        autostartLink = virDomainConfigFile(BHYVE_AUTOSTART_DIR, vm->def->name);

        if (autostart) {
            if (g_mkdir_with_parents(BHYVE_AUTOSTART_DIR, 0777) < 0) {
                virReportSystemError(errno,
                                     _("cannot create autostart directory %1$s"),
                                     BHYVE_AUTOSTART_DIR);
                goto cleanup;
            }

            if (symlink(configFile, autostartLink) < 0) {
                virReportSystemError(errno,
                                     _("Failed to create symlink '%1$s' to '%2$s'"),
                                     autostartLink, configFile);
                goto cleanup;
            }
        } else {
            if (unlink(autostartLink) < 0 && errno != ENOENT && errno != ENOTDIR) {
                virReportSystemError(errno,
                                     _("Failed to delete symlink '%1$s'"),
                                     autostartLink);
                goto cleanup;
            }
        }

        vm->autostart = autostart;
    }

    ret = 0;

 cleanup:
    VIR_FREE(configFile);
    VIR_FREE(autostartLink);
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainIsActive(virDomainPtr domain)
{
    virDomainObj *obj;
    int ret = -1;

    if (!(obj = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainIsActiveEnsureACL(domain->conn, obj->def) < 0)
        goto cleanup;

    ret = virDomainObjIsActive(obj);

 cleanup:
    virDomainObjEndAPI(&obj);
    return ret;
}

static int
bhyveDomainIsPersistent(virDomainPtr domain)
{
    virDomainObj *obj;
    int ret = -1;

    if (!(obj = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainIsPersistentEnsureACL(domain->conn, obj->def) < 0)
        goto cleanup;

    ret = obj->persistent;

 cleanup:
    virDomainObjEndAPI(&obj);
    return ret;
}

static char *
bhyveDomainGetOSType(virDomainPtr dom)
{
    virDomainObj *vm;
    char *ret = NULL;

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (virDomainGetOSTypeEnsureACL(dom->conn, vm->def) < 0)
        goto cleanup;

    ret = g_strdup(virDomainOSTypeToString(vm->def->os.type));

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static char *
bhyveDomainGetXMLDesc(virDomainPtr domain, unsigned int flags)
{
    struct _bhyveConn *privconn = domain->conn->privateData;
    virDomainObj *vm;
    char *ret = NULL;

    virCheckFlags(VIR_DOMAIN_XML_COMMON_FLAGS, NULL);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainGetXMLDescEnsureACL(domain->conn, vm->def, flags) < 0)
        goto cleanup;

    ret = virDomainDefFormat(vm->def, privconn->xmlopt,
                             virDomainDefFormatConvertXMLFlags(flags));

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static virDomainPtr
bhyveDomainDefineXMLFlags(virConnectPtr conn, const char *xml, unsigned int flags)
{
    struct _bhyveConn *privconn = conn->privateData;
    virDomainPtr dom = NULL;
    g_autoptr(virDomainDef) def = NULL;
    g_autoptr(virDomainDef) oldDef = NULL;
    virDomainObj *vm = NULL;
    virObjectEvent *event = NULL;
    g_autoptr(virCaps) caps = NULL;
    unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE;

    virCheckFlags(VIR_DOMAIN_DEFINE_VALIDATE, NULL);

    if (flags & VIR_DOMAIN_DEFINE_VALIDATE)
        parse_flags |= VIR_DOMAIN_DEF_PARSE_VALIDATE_SCHEMA;

    caps = bhyveDriverGetCapabilities(privconn);
    if (!caps)
        return NULL;

    /* Avoid parsing the whole domain definition for ACL checks */
    if (!(def = virDomainDefIDsParseString(xml, privconn->xmlopt, parse_flags)))
        return NULL;

    if (virDomainDefineXMLFlagsEnsureACL(conn, def) < 0)
        return NULL;

    g_clear_pointer(&def, virDomainDefFree);

    if ((def = virDomainDefParseString(xml, privconn->xmlopt,
                                       NULL, parse_flags)) == NULL)
        goto cleanup;

    if (virXMLCheckIllegalChars("name", def->name, "\n") < 0)
        goto cleanup;

    if (bhyveDomainAssignAddresses(def, NULL) < 0)
        goto cleanup;

    if (!(vm = virDomainObjListAdd(privconn->domains, &def,
                                   privconn->xmlopt,
                                   0, &oldDef)))
        goto cleanup;
    vm->persistent = 1;

    if (virDomainDefSave(vm->newDef ? vm->newDef : vm->def,
                         privconn->xmlopt, BHYVE_CONFIG_DIR) < 0) {
        virDomainObjListRemove(privconn->domains, vm);
        goto cleanup;
    }

    event = virDomainEventLifecycleNewFromObj(vm,
                                              VIR_DOMAIN_EVENT_DEFINED,
                                              !oldDef ?
                                              VIR_DOMAIN_EVENT_DEFINED_ADDED :
                                              VIR_DOMAIN_EVENT_DEFINED_UPDATED);

    dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id);

 cleanup:
    virDomainObjEndAPI(&vm);
    virObjectEventStateQueue(privconn->domainEventState, event);

    return dom;
}

static virDomainPtr
bhyveDomainDefineXML(virConnectPtr conn, const char *xml)
{
    return bhyveDomainDefineXMLFlags(conn, xml, 0);
}

static int
bhyveDomainUndefineFlags(virDomainPtr domain, unsigned int flags)
{
    struct _bhyveConn *privconn = domain->conn->privateData;
    virObjectEvent *event = NULL;
    virDomainObj *vm;
    g_autofree char *nvram_path = NULL;
    int ret = -1;

    virCheckFlags(VIR_DOMAIN_UNDEFINE_NVRAM |
                  VIR_DOMAIN_UNDEFINE_KEEP_NVRAM, -1);

    if ((flags & VIR_DOMAIN_UNDEFINE_NVRAM) &&
        (flags & VIR_DOMAIN_UNDEFINE_KEEP_NVRAM)) {
        virReportError(VIR_ERR_OPERATION_INVALID,
                       "%s", _("cannot both keep and delete nvram"));
        return -1;
    }

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainUndefineFlagsEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (!vm->persistent) {
        virReportError(VIR_ERR_OPERATION_INVALID,
                       "%s", _("Cannot undefine transient domain"));
        goto cleanup;
    }

    if (vm->def->os.loader && vm->def->os.loader->nvram &&
        virStorageSourceIsLocalStorage(vm->def->os.loader->nvram)) {
        nvram_path = g_strdup(vm->def->os.loader->nvram->path);
    }

    if (nvram_path && virFileExists(nvram_path)) {
        if ((flags & VIR_DOMAIN_UNDEFINE_NVRAM)) {
            if (unlink(nvram_path) < 0) {
                virReportSystemError(errno,
                                     _("failed to remove nvram: %1$s"),
                                     nvram_path);
                goto cleanup;
            }
        } else if (!(flags & VIR_DOMAIN_UNDEFINE_KEEP_NVRAM)) {
            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
                           _("cannot undefine domain with nvram"));
            goto cleanup;
        }
    }

    if (virDomainDeleteConfig(BHYVE_CONFIG_DIR,
                              BHYVE_AUTOSTART_DIR,
                              vm) < 0)
        goto cleanup;

    event = virDomainEventLifecycleNewFromObj(vm,
                                              VIR_DOMAIN_EVENT_UNDEFINED,
                                              VIR_DOMAIN_EVENT_UNDEFINED_REMOVED);

    if (virDomainObjIsActive(vm))
        vm->persistent = 0;
    else
        virDomainObjListRemove(privconn->domains, vm);

    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    virObjectEventStateQueue(privconn->domainEventState, event);
    return ret;
}

static int
bhyveDomainUndefine(virDomainPtr domain)
{
    return bhyveDomainUndefineFlags(domain, 0);
}

static int
bhyveConnectListDomains(virConnectPtr conn, int *ids, int maxids)
{
    struct _bhyveConn *privconn = conn->privateData;

    if (virConnectListDomainsEnsureACL(conn) < 0)
        return -1;

    return virDomainObjListGetActiveIDs(privconn->domains, ids, maxids,
                                        virConnectListDomainsCheckACL, conn);
}

static int
bhyveConnectNumOfDomains(virConnectPtr conn)
{
    struct _bhyveConn *privconn = conn->privateData;

    if (virConnectNumOfDomainsEnsureACL(conn) < 0)
        return -1;

    return virDomainObjListNumOfDomains(privconn->domains, true,
                                        virConnectNumOfDomainsCheckACL, conn);
}

static int
bhyveConnectListDefinedDomains(virConnectPtr conn, char **const names,
                               int maxnames)
{
    struct _bhyveConn *privconn = conn->privateData;

    if (virConnectListDefinedDomainsEnsureACL(conn) < 0)
        return -1;

    memset(names, 0, sizeof(*names) * maxnames);
    return virDomainObjListGetInactiveNames(privconn->domains, names,
                                            maxnames,
                                            virConnectListDefinedDomainsCheckACL,
                                            conn);
}

static int
bhyveConnectNumOfDefinedDomains(virConnectPtr conn)
{
    struct _bhyveConn *privconn = conn->privateData;

    if (virConnectNumOfDefinedDomainsEnsureACL(conn) < 0)
        return -1;

    return virDomainObjListNumOfDomains(privconn->domains, false,
                                        virConnectNumOfDefinedDomainsCheckACL,
                                        conn);
}

static char *
bhyveConnectDomainXMLToNative(virConnectPtr conn,
                              const char *format,
                              const char *xmlData,
                              unsigned int flags)
{
    g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
    struct _bhyveConn *privconn = conn->privateData;
    g_autoptr(virDomainDef) def = NULL;
    g_autoptr(virCommand) cmd = NULL;
    g_autoptr(virCommand) loadcmd = NULL;

    virCheckFlags(0, NULL);

    if (virConnectDomainXMLToNativeEnsureACL(conn) < 0)
        return NULL;

    if (STRNEQ(format, BHYVE_CONFIG_FORMAT_ARGV)) {
        virReportError(VIR_ERR_INVALID_ARG,
                       _("Unsupported config type %1$s"), format);
        return NULL;
    }

    if (!(def = virDomainDefParseString(xmlData, privconn->xmlopt,
                                        NULL, VIR_DOMAIN_DEF_PARSE_INACTIVE)))
        return NULL;

    if (bhyveDomainAssignAddresses(def, NULL) < 0)
        return NULL;

    if (def->os.bootloader == NULL &&
        def->os.loader) {

        if (!virDomainDefHasOldStyleROUEFI(def)) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Only read-only pflash is supported."));
            return NULL;
        }

        if ((bhyveDriverGetBhyveCaps(privconn) & BHYVE_CAP_LPC_BOOTROM) == 0) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Installed bhyve binary does not support bootrom"));
            return NULL;
        }
    } else {
        if (!(loadcmd = virBhyveProcessBuildLoadCmd(privconn, def,
                                                    "<device.map>", NULL)))
            return NULL;

        virCommandToStringBuf(loadcmd, &buf, false, false);
        virBufferAddChar(&buf, '\n');
    }

    if (!(cmd = virBhyveProcessBuildBhyveCmd(privconn, def, true)))
        return NULL;

    virCommandToStringBuf(cmd, &buf, false, false);

    return virBufferContentAndReset(&buf);
}

static int
bhyveConnectListAllDomains(virConnectPtr conn,
                           virDomainPtr **domains,
                           unsigned int flags)
{
    struct _bhyveConn *privconn = conn->privateData;

    virCheckFlags(VIR_CONNECT_LIST_DOMAINS_FILTERS_ALL, -1);

    if (virConnectListAllDomainsEnsureACL(conn) < 0)
        return -1;

    return virDomainObjListExport(privconn->domains, conn, domains,
                                  virConnectListAllDomainsCheckACL, flags);
}

static virDomainPtr
bhyveDomainLookupByUUID(virConnectPtr conn,
                        const unsigned char *uuid)
{
    struct _bhyveConn *privconn = conn->privateData;
    virDomainObj *vm;
    virDomainPtr dom = NULL;

    vm = virDomainObjListFindByUUID(privconn->domains, uuid);

    if (!vm) {
        char uuidstr[VIR_UUID_STRING_BUFLEN];
        virUUIDFormat(uuid, uuidstr);
        virReportError(VIR_ERR_NO_DOMAIN,
                       _("No domain with matching uuid '%1$s'"), uuidstr);
        goto cleanup;
    }

    if (virDomainLookupByUUIDEnsureACL(conn, vm->def) < 0)
        goto cleanup;

    dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id);

 cleanup:
    virDomainObjEndAPI(&vm);
    return dom;
}

static virDomainPtr bhyveDomainLookupByName(virConnectPtr conn,
                                            const char *name)
{
    struct _bhyveConn *privconn = conn->privateData;
    virDomainObj *vm;
    virDomainPtr dom = NULL;

    vm = virDomainObjListFindByName(privconn->domains, name);

    if (!vm) {
        virReportError(VIR_ERR_NO_DOMAIN,
                       _("no domain with matching name '%1$s'"), name);
        goto cleanup;
    }

    if (virDomainLookupByNameEnsureACL(conn, vm->def) < 0)
        goto cleanup;

    dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id);

 cleanup:
    virDomainObjEndAPI(&vm);
    return dom;
}

static virDomainPtr
bhyveDomainLookupByID(virConnectPtr conn,
                      int id)
{
    struct _bhyveConn *privconn = conn->privateData;
    virDomainObj *vm;
    virDomainPtr dom = NULL;

    vm = virDomainObjListFindByID(privconn->domains, id);

    if (!vm) {
        virReportError(VIR_ERR_NO_DOMAIN,
                       _("No domain with matching ID '%1$d'"), id);
        goto cleanup;
    }

    if (virDomainLookupByIDEnsureACL(conn, vm->def) < 0)
        goto cleanup;

    dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id);

 cleanup:
    virDomainObjEndAPI(&vm);
    return dom;
}

static int
bhyveDomainCreateWithFlags(virDomainPtr dom,
                           unsigned int flags)
{
    struct _bhyveConn *privconn = dom->conn->privateData;
    virDomainObj *vm;
    virObjectEvent *event = NULL;
    unsigned int start_flags = 0;
    int ret = -1;

    virCheckFlags(VIR_DOMAIN_START_AUTODESTROY, -1);

    if (flags & VIR_DOMAIN_START_AUTODESTROY)
        start_flags |= VIR_BHYVE_PROCESS_START_AUTODESTROY;

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (virDomainCreateWithFlagsEnsureACL(dom->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjIsActive(vm)) {
        virReportError(VIR_ERR_OPERATION_INVALID,
                       "%s", _("Domain is already running"));
        goto cleanup;
    }

    ret = virBhyveProcessStart(privconn, dom->conn, vm,
                               VIR_DOMAIN_RUNNING_BOOTED,
                               start_flags);

    if (ret == 0)
        event = virDomainEventLifecycleNewFromObj(vm,
                                                  VIR_DOMAIN_EVENT_STARTED,
                                                  VIR_DOMAIN_EVENT_STARTED_BOOTED);

 cleanup:
    virDomainObjEndAPI(&vm);
    virObjectEventStateQueue(privconn->domainEventState, event);
    return ret;
}

static int
bhyveDomainCreate(virDomainPtr dom)
{
    return bhyveDomainCreateWithFlags(dom, 0);
}

static virDomainPtr
bhyveDomainCreateXML(virConnectPtr conn,
                     const char *xml,
                     unsigned int flags)
{
    struct _bhyveConn *privconn = conn->privateData;
    virDomainPtr dom = NULL;
    g_autoptr(virDomainDef) def = NULL;
    virDomainObj *vm = NULL;
    virObjectEvent *event = NULL;
    unsigned int start_flags = 0;
    unsigned int parse_flags = VIR_DOMAIN_DEF_PARSE_INACTIVE;

    virCheckFlags(VIR_DOMAIN_START_AUTODESTROY |
                  VIR_DOMAIN_START_VALIDATE, NULL);

    if (flags & VIR_DOMAIN_START_VALIDATE)
        parse_flags |= VIR_DOMAIN_DEF_PARSE_VALIDATE_SCHEMA;
    if (flags & VIR_DOMAIN_START_AUTODESTROY)
        start_flags |= VIR_BHYVE_PROCESS_START_AUTODESTROY;

    /* Avoid parsing the whole domain definition for ACL checks */
    if (!(def = virDomainDefIDsParseString(xml, privconn->xmlopt, parse_flags)))
        return NULL;

    if (virDomainCreateXMLEnsureACL(conn, def) < 0)
        return NULL;

    g_clear_pointer(&def, virDomainDefFree);

    if ((def = virDomainDefParseString(xml, privconn->xmlopt,
                                       NULL, parse_flags)) == NULL)
        goto cleanup;

    if (bhyveDomainAssignAddresses(def, NULL) < 0)
        goto cleanup;

    if (!(vm = virDomainObjListAdd(privconn->domains, &def,
                                   privconn->xmlopt,
                                   VIR_DOMAIN_OBJ_LIST_ADD_LIVE |
                                   VIR_DOMAIN_OBJ_LIST_ADD_CHECK_LIVE, NULL)))
        goto cleanup;

    if (virBhyveProcessStart(privconn, conn, vm,
                             VIR_DOMAIN_RUNNING_BOOTED,
                             start_flags) < 0) {
        /* If domain is not persistent, remove its data */
        if (!vm->persistent)
            virDomainObjListRemove(privconn->domains, vm);
        goto cleanup;
    }

    event = virDomainEventLifecycleNewFromObj(vm,
                                              VIR_DOMAIN_EVENT_STARTED,
                                              VIR_DOMAIN_EVENT_STARTED_BOOTED);

    dom = virGetDomain(conn, vm->def->name, vm->def->uuid, vm->def->id);

 cleanup:
    virDomainObjEndAPI(&vm);
    virObjectEventStateQueue(privconn->domainEventState, event);

    return dom;
}

static int
bhyveDomainDestroyFlags(virDomainPtr dom, unsigned int flags)
{
    virConnectPtr conn = dom->conn;
    struct _bhyveConn *privconn = conn->privateData;
    virDomainObj *vm;
    virObjectEvent *event = NULL;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (virDomainDestroyFlagsEnsureACL(conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    ret = virBhyveProcessStop(privconn, vm, VIR_DOMAIN_SHUTOFF_DESTROYED);
    event = virDomainEventLifecycleNewFromObj(vm,
                                              VIR_DOMAIN_EVENT_STOPPED,
                                              VIR_DOMAIN_EVENT_STOPPED_DESTROYED);

    if (!vm->persistent)
        virDomainObjListRemove(privconn->domains, vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    virObjectEventStateQueue(privconn->domainEventState, event);
    return ret;
}

static int
bhyveDomainDestroy(virDomainPtr dom)
{
    return bhyveDomainDestroyFlags(dom, 0);
}

static int
bhyveDomainShutdownSignal(virDomainObj *vm,
                          bool isReboot)
{
    bhyveDomainObjPrivate *priv;

    if (isReboot) {
        priv = vm->privateData;
        bhyveMonitorSetReboot(priv->mon);
    }

    return virBhyveProcessShutdown(vm);
}

static int
bhyveDomainShutdownFlagsAgent(virDomainObj *vm,
                              bool isReboot,
                              bool reportError)
{
    int ret = -1;
    qemuAgent *agent;
    int agentFlag = isReboot ? QEMU_AGENT_SHUTDOWN_REBOOT :
        QEMU_AGENT_SHUTDOWN_POWERDOWN;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
        return -1;

    if (virDomainObjGetState(vm, NULL) != VIR_DOMAIN_RUNNING) {
        virReportError(VIR_ERR_OPERATION_INVALID,
                       "%s", _("domain is not running"));
        goto endjob;
    }

    if (bhyveDomainEnsureAgent(vm, reportError) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    qemuAgentShutdown(agent, agentFlag);
    bhyveDomainObjExitAgent(vm, agent);
    ret = 0;

 endjob:
    virDomainObjEndAgentJob(vm);
    return ret;
}

static int
bhyveDomainRebootAgent(virDomainObj *vm, bool isReboot, bool reportError)
{
    return bhyveDomainShutdownFlagsAgent(vm, isReboot, reportError);
}

static int
bhyveDomainShutdownFlags(virDomainPtr dom, unsigned int flags)
{
    virDomainObj *vm;
    bhyveDomainObjPrivate *priv;
    int ret = -1;
    bool isReboot = false;
    bool useAgent = false;
    bool agentRequested, signalRequested;
    bool agentForced;

    virCheckFlags(VIR_DOMAIN_SHUTDOWN_SIGNAL |
                  VIR_DOMAIN_SHUTDOWN_GUEST_AGENT, -1);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (vm->def->onPoweroff == VIR_DOMAIN_LIFECYCLE_ACTION_RESTART ||
        vm->def->onPoweroff == VIR_DOMAIN_LIFECYCLE_ACTION_RESTART_RENAME) {
        isReboot = true;
        VIR_INFO("Domain on_poweroff setting overridden, attempting reboot");
    }

    priv = vm->privateData;
    agentRequested = flags & VIR_DOMAIN_SHUTDOWN_GUEST_AGENT;
    signalRequested = flags & VIR_DOMAIN_SHUTDOWN_SIGNAL;

    /* Prefer agent unless we were requested to not to. */
    if (agentRequested || !flags)
        useAgent = true;

    if (virDomainShutdownFlagsEnsureACL(dom->conn, vm->def, flags) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    agentForced = agentRequested && !signalRequested;
    if (useAgent) {
        ret = bhyveDomainShutdownFlagsAgent(vm, isReboot, agentForced);
        if (((ret < 0) || (priv->agent != NULL)) && agentForced)
            goto cleanup;
    }

    /* If we are not enforced to use just an agent, try signal
     * shutdown as well in case agent did not succeed.
     */
    if (!useAgent || (((ret < 0) ||
        (priv->agent != NULL)) && (signalRequested || !flags))) {
        /* Even if agent failed, we have to check if guest went away
         * by itself while our locks were down.  */
        if (useAgent && !virDomainObjIsActive(vm)) {
            ret = 0;
            goto cleanup;
        }

        ret = bhyveDomainShutdownSignal(vm, isReboot);
    }

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainShutdown(virDomainPtr dom)
{
    return bhyveDomainShutdownFlags(dom, 0);
}

static int
bhyveDomainReboot(virDomainPtr dom, unsigned int flags)
{
    virConnectPtr conn = dom->conn;
    virDomainObj *vm;
    bhyveDomainObjPrivate *priv;
    int ret = -1;
    bool isReboot = true;
    bool useAgent = false;
    bool agentRequested, signalRequested;
    bool agentForced;

    virCheckFlags(VIR_DOMAIN_REBOOT_SIGNAL |
                  VIR_DOMAIN_REBOOT_GUEST_AGENT, -1);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (vm->def->onReboot == VIR_DOMAIN_LIFECYCLE_ACTION_DESTROY ||
        vm->def->onReboot == VIR_DOMAIN_LIFECYCLE_ACTION_PRESERVE) {
        isReboot = false;
        VIR_INFO("Domain on_reboot setting overridden, shutting down");
    }

    priv = vm->privateData;
    agentRequested = flags & VIR_DOMAIN_REBOOT_GUEST_AGENT;
    signalRequested = flags & VIR_DOMAIN_REBOOT_SIGNAL;

    /* Prefer agent unless we were requested to not to. */
    if (agentRequested || !flags)
        useAgent = true;

    if (virDomainRebootEnsureACL(conn, vm->def, flags) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    agentForced = agentRequested && !signalRequested;
    if (useAgent) {
        ret = bhyveDomainRebootAgent(vm, isReboot, agentForced);
        if (((ret < 0) || (priv->agent != NULL)) && agentForced)
            goto cleanup;
    }

    /* If we are not enforced to use just an agent, try signal
     * reboot as well in case agent did not succeed.
     */
    if (!useAgent || (((ret < 0) ||
        (priv->agent != NULL)) && (signalRequested || !flags))) {
        ret = bhyveDomainShutdownSignal(vm, isReboot);
    }

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainOpenConsole(virDomainPtr dom,
                       const char *dev_name G_GNUC_UNUSED,
                       virStreamPtr st,
                       unsigned int flags)
{
    virDomainObj *vm = NULL;
    virDomainChrDef *chr = NULL;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        goto cleanup;

    if (virDomainOpenConsoleEnsureACL(dom->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    if (!vm->def->nserials) {
        virReportError(VIR_ERR_INTERNAL_ERROR,
                       "%s", _("no console devices available"));
        goto cleanup;
    }

    chr = vm->def->serials[0];

    if (virFDStreamOpenPTY(st, chr->source->data.nmdm.slave,
                           0, 0, O_RDWR) < 0)
        goto cleanup;

    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainSetMetadata(virDomainPtr dom,
                       int type,
                       const char *metadata,
                       const char *key,
                       const char *uri,
                       unsigned int flags)
{
    virConnectPtr conn = dom->conn;
    struct _bhyveConn *privconn = conn->privateData;
    virDomainObj *vm;
    int ret = -1;

    virCheckFlags(VIR_DOMAIN_AFFECT_LIVE |
                  VIR_DOMAIN_AFFECT_CONFIG, -1);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        return -1;

    if (virDomainSetMetadataEnsureACL(conn, vm->def, flags) < 0)
        goto cleanup;

    ret = virDomainObjSetMetadata(vm, type, metadata, key, uri,
                                  privconn->xmlopt, BHYVE_STATE_DIR,
                                  BHYVE_CONFIG_DIR, flags);

    if (ret == 0) {
        virObjectEvent *ev = NULL;
        ev = virDomainEventMetadataChangeNewFromObj(vm, type, uri);
        virObjectEventStateQueue(privconn->domainEventState, ev);
    }


 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static char *
bhyveDomainGetMetadata(virDomainPtr dom,
                      int type,
                      const char *uri,
                      unsigned int flags)
{
    virDomainObj *vm;
    char *ret = NULL;

    if (!(vm = bhyveDomObjFromDomain(dom)))
        return NULL;

    if (virDomainGetMetadataEnsureACL(dom->conn, vm->def) < 0)
        goto cleanup;

    ret = virDomainObjGetMetadata(vm, type, uri, flags);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveNodeGetCPUStats(virConnectPtr conn,
                     int cpuNum,
                     virNodeCPUStatsPtr params,
                     int *nparams,
                     unsigned int flags)
{
    virCheckFlags(0, -1);

    if (virNodeGetCPUStatsEnsureACL(conn) < 0)
        return -1;

    return virHostCPUGetStats(cpuNum, params, nparams);
}

static int
bhyveNodeGetMemoryStats(virConnectPtr conn,
                        int cellNum,
                        virNodeMemoryStatsPtr params,
                        int *nparams,
                        unsigned int flags)
{
    virCheckFlags(0, -1);

    if (virNodeGetMemoryStatsEnsureACL(conn) < 0)
        return -1;

    return virHostMemGetStats(cellNum, params, nparams);
}

static int
bhyveNodeGetInfo(virConnectPtr conn,
                 virNodeInfoPtr nodeinfo)
{
    if (virNodeGetInfoEnsureACL(conn) < 0)
        return -1;

    return virCapabilitiesGetNodeInfo(nodeinfo);
}

static int
bhyveStateCleanup(void)
{
    VIR_DEBUG("bhyve state cleanup");

    if (bhyve_driver == NULL)
        return -1;

    virObjectUnref(bhyve_driver->domains);
    virObjectUnref(bhyve_driver->caps);
    virObjectUnref(bhyve_driver->xmlopt);
    virSysinfoDefFree(bhyve_driver->hostsysinfo);
    virObjectUnref(bhyve_driver->domainEventState);
    virObjectUnref(bhyve_driver->config);
    virPortAllocatorRangeFree(bhyve_driver->remotePorts);

    if (bhyve_driver->lockFD != -1)
        virPidFileRelease(BHYVE_STATE_DIR, "driver", bhyve_driver->lockFD);

    virMutexDestroy(&bhyve_driver->lock);
    VIR_FREE(bhyve_driver);

    return 0;
}

static virDrvStateInitResult
bhyveStateInitialize(bool privileged,
                     const char *root,
                     bool monolithic G_GNUC_UNUSED,
                     virStateInhibitCallback callback G_GNUC_UNUSED,
                     void *opaque G_GNUC_UNUSED)
{
    virDomainDriverAutoStartConfig autostartCfg;

    if (root != NULL) {
        virReportError(VIR_ERR_INVALID_ARG, "%s",
                       _("Driver does not support embedded mode"));
        return -1;
    }

    if (!privileged) {
        VIR_INFO("Not running privileged, disabling driver");
        return VIR_DRV_STATE_INIT_SKIPPED;
    }

    bhyve_driver = g_new0(bhyveConn, 1);

    bhyve_driver->lockFD = -1;
    if (virMutexInit(&bhyve_driver->lock) < 0) {
        VIR_FREE(bhyve_driver);
        return VIR_DRV_STATE_INIT_ERROR;
    }

    if (!(bhyve_driver->caps = virBhyveCapsBuild()))
        goto cleanup;

    if (virBhyveProbeCaps(&bhyve_driver->bhyvecaps) < 0)
        goto cleanup;

    if (virBhyveProbeGrubCaps(&bhyve_driver->grubcaps) < 0)
        goto cleanup;

    if (!(bhyve_driver->xmlopt = virBhyveDriverCreateXMLConf(bhyve_driver)))
        goto cleanup;

    if (!(bhyve_driver->domains = virDomainObjListNew()))
        goto cleanup;

    if (!(bhyve_driver->domainEventState = virObjectEventStateNew()))
        goto cleanup;

    if (!(bhyve_driver->remotePorts = virPortAllocatorRangeNew(_("display"),
                                                               5900, 65535)))
        goto cleanup;

    bhyve_driver->hostsysinfo = virSysinfoRead();

    if (!(bhyve_driver->config = virBhyveDriverConfigNew()))
        goto cleanup;

    if (virBhyveLoadDriverConfig(bhyve_driver->config, SYSCONFDIR "/libvirt/bhyve.conf") < 0)
        goto cleanup;

    if (g_mkdir_with_parents(BHYVE_LOG_DIR, 0777) < 0) {
        virReportSystemError(errno,
                             _("Failed to mkdir %1$s"),
                             BHYVE_LOG_DIR);
        goto cleanup;
    }

    if (g_mkdir_with_parents(BHYVE_STATE_DIR, 0777) < 0) {
        virReportSystemError(errno,
                             _("Failed to mkdir %1$s"),
                             BHYVE_STATE_DIR);
        goto cleanup;
    }

    if ((bhyve_driver->lockFD =
         virPidFileAcquire(BHYVE_STATE_DIR, "driver", getpid())) < 0)
        goto cleanup;

    if (virDomainObjListLoadAllConfigs(bhyve_driver->domains,
                                       BHYVE_STATE_DIR,
                                       NULL, true,
                                       bhyve_driver->xmlopt,
                                       NULL, NULL) < 0)
        goto cleanup;

    if (virDomainObjListLoadAllConfigs(bhyve_driver->domains,
                                       BHYVE_CONFIG_DIR,
                                       BHYVE_AUTOSTART_DIR, false,
                                       bhyve_driver->xmlopt,
                                       NULL, NULL) < 0)
        goto cleanup;

    virBhyveProcessReconnectAll(bhyve_driver);

    autostartCfg = (virDomainDriverAutoStartConfig) {
        .stateDir = BHYVE_STATE_DIR,
        .callback = bhyveAutostartDomain,
        .opaque = bhyve_driver,
    };
    virDomainDriverAutoStart(bhyve_driver->domains, &autostartCfg);

    return VIR_DRV_STATE_INIT_COMPLETE;

 cleanup:
    bhyveStateCleanup();
    return VIR_DRV_STATE_INIT_ERROR;
}

unsigned
bhyveDriverGetBhyveCaps(struct _bhyveConn *driver)
{
    if (driver != NULL)
        return driver->bhyvecaps;
    return 0;
}

unsigned
bhyveDriverGetGrubCaps(struct _bhyveConn *driver)
{
    if (driver != NULL)
        return driver->grubcaps;
    return 0;
}

static int
bhyveConnectGetMaxVcpus(virConnectPtr conn,
                        const char *type)
{
    if (virConnectGetMaxVcpusEnsureACL(conn) < 0)
        return -1;

    /*
     * Bhyve supports up to 16 VCPUs, but offers no method to check this
     * value. Hardcode 16...
     */
    if (!type || STRCASEEQ(type, "bhyve"))
        return 16;

    virReportError(VIR_ERR_INVALID_ARG, _("unknown type '%1$s'"), type);
    return -1;
}

static unsigned long long
bhyveNodeGetFreeMemory(virConnectPtr conn)
{
    unsigned long long freeMem;

    if (virNodeGetFreeMemoryEnsureACL(conn) < 0)
        return 0;

    if (virHostMemGetInfo(NULL, &freeMem) < 0)
        return 0;

    return freeMem;
}

static int
bhyveNodeGetCPUMap(virConnectPtr conn,
                   unsigned char **cpumap,
                   unsigned int *online,
                   unsigned int flags)
{
    virCheckFlags(0, -1);

    if (virNodeGetCPUMapEnsureACL(conn) < 0)
        return -1;

    return virHostCPUGetMap(cpumap, online);
}

static int
bhyveNodeGetMemoryParameters(virConnectPtr conn,
                             virTypedParameterPtr params,
                             int *nparams,
                             unsigned int flags)
{
    virCheckFlags(VIR_TYPED_PARAM_STRING_OKAY, -1);

    if (virNodeGetMemoryParametersEnsureACL(conn) < 0)
        return -1;

    return virHostMemGetParameters(params, nparams);
}

static int
bhyveNodeSetMemoryParameters(virConnectPtr conn,
                             virTypedParameterPtr params,
                             int nparams,
                             unsigned int flags)
{
    virCheckFlags(0, -1);

    if (virNodeSetMemoryParametersEnsureACL(conn) < 0)
        return -1;

    return virHostMemSetParameters(params, nparams);
}

static char *
bhyveConnectBaselineCPU(virConnectPtr conn,
                        const char **xmlCPUs,
                        unsigned int ncpus,
                        unsigned int flags)
{
    virCPUDef **cpus = NULL;
    virCPUDef *cpu = NULL;
    char *cpustr = NULL;

    virCheckFlags(VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES |
                  VIR_CONNECT_BASELINE_CPU_MIGRATABLE, NULL);

    if (virConnectBaselineCPUEnsureACL(conn) < 0)
        goto cleanup;

    if (!(cpus = virCPUDefListParse(xmlCPUs, ncpus, VIR_CPU_TYPE_HOST)))
        goto cleanup;

    if (!(cpu = virCPUBaseline(VIR_ARCH_NONE, cpus, ncpus, NULL, NULL,
                               !!(flags & VIR_CONNECT_BASELINE_CPU_MIGRATABLE))))
        goto cleanup;

    if ((flags & VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES) &&
        virCPUExpandFeatures(cpus[0]->arch, cpu) < 0)
        goto cleanup;

    cpustr = virCPUDefFormat(cpu, NULL);

 cleanup:
    virCPUDefListFree(cpus);
    virCPUDefFree(cpu);

    return cpustr;
}

static int
bhyveConnectCompareCPU(virConnectPtr conn,
                       const char *xmlDesc,
                       unsigned int flags)
{
    struct _bhyveConn *driver = conn->privateData;
    g_autoptr(virCaps) caps = NULL;
    bool failIncompatible;
    bool validateXML;

    virCheckFlags(VIR_CONNECT_COMPARE_CPU_FAIL_INCOMPATIBLE |
                  VIR_CONNECT_COMPARE_CPU_VALIDATE_XML,
                  VIR_CPU_COMPARE_ERROR);

    if (virConnectCompareCPUEnsureACL(conn) < 0)
        return VIR_CPU_COMPARE_ERROR;

    failIncompatible = !!(flags & VIR_CONNECT_COMPARE_CPU_FAIL_INCOMPATIBLE);
    validateXML = !!(flags & VIR_CONNECT_COMPARE_CPU_VALIDATE_XML);

    if (!(caps = bhyveDriverGetCapabilities(driver)))
        return VIR_CPU_COMPARE_ERROR;

    if (!caps->host.cpu ||
        !caps->host.cpu->model) {
        if (failIncompatible) {
            virReportError(VIR_ERR_CPU_INCOMPATIBLE, "%s",
                           _("cannot get host CPU capabilities"));
            return VIR_CPU_COMPARE_ERROR;
        }
        VIR_WARN("cannot get host CPU capabilities");
        return VIR_CPU_COMPARE_INCOMPATIBLE;
    }

    return virCPUCompareXML(caps->host.arch, caps->host.cpu,
                            xmlDesc, failIncompatible, validateXML);
}

static int
bhyveConnectDomainEventRegisterAny(virConnectPtr conn,
                                   virDomainPtr dom,
                                   int eventID,
                                   virConnectDomainEventGenericCallback callback,
                                   void *opaque,
                                   virFreeCallback freecb)
{
    struct _bhyveConn *privconn = conn->privateData;
    int ret;

    if (virConnectDomainEventRegisterAnyEnsureACL(conn) < 0)
        return -1;

    if (virDomainEventStateRegisterID(conn,
                                      privconn->domainEventState,
                                      dom, eventID,
                                      callback, opaque, freecb, &ret) < 0)
        ret = -1;

    return ret;
}

static int
bhyveConnectDomainEventDeregisterAny(virConnectPtr conn,
                                     int callbackID)
{
    struct _bhyveConn *privconn = conn->privateData;

    if (virConnectDomainEventDeregisterAnyEnsureACL(conn) < 0)
        return -1;

    if (virObjectEventStateDeregisterID(conn,
                                        privconn->domainEventState,
                                        callbackID, true) < 0)
        return -1;

    return 0;
}

static int
bhyveDomainHasManagedSaveImage(virDomainPtr domain, unsigned int flags)
{
    virDomainObj *vm = NULL;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainHasManagedSaveImageEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static const char *
bhyveConnectGetType(virConnectPtr conn)
{
    if (virConnectGetTypeEnsureACL(conn) < 0)
        return NULL;

    return "BHYVE";
}

static int bhyveConnectIsAlive(virConnectPtr conn G_GNUC_UNUSED)
{
    return 1;
}

static int
bhyveConnectIsSecure(virConnectPtr conn G_GNUC_UNUSED)
{
    /* Trivially secure, since always inside the daemon */
    return 1;
}

static int
bhyveConnectIsEncrypted(virConnectPtr conn G_GNUC_UNUSED)
{
    /* Not encrypted, but remote driver takes care of that */
    return 0;
}

static char *
bhyveConnectDomainXMLFromNative(virConnectPtr conn,
                                const char *nativeFormat,
                                const char *nativeConfig,
                                unsigned int flags)
{
    g_autoptr(virDomainDef) def = NULL;
    struct _bhyveConn *privconn = conn->privateData;
    unsigned bhyveCaps = bhyveDriverGetBhyveCaps(privconn);

    virCheckFlags(0, NULL);

    if (virConnectDomainXMLFromNativeEnsureACL(conn) < 0)
        return NULL;

    if (STRNEQ(nativeFormat, BHYVE_CONFIG_FORMAT_ARGV)) {
        virReportError(VIR_ERR_INVALID_ARG,
                       _("unsupported config type %1$s"), nativeFormat);
        return NULL;
    }

    def = bhyveParseCommandLineString(nativeConfig, bhyveCaps,
                                      privconn->xmlopt);
    if (def == NULL)
        return NULL;

    return virDomainDefFormat(def, privconn->xmlopt, 0);
}

static char *
bhyveConnectGetDomainCapabilities(virConnectPtr conn,
                                  const char *emulatorbin,
                                  const char *arch_str,
                                  const char *machine,
                                  const char *virttype_str,
                                  unsigned int flags)
{
    virDomainCaps *caps = NULL;
    char *ret = NULL;
    int virttype = VIR_DOMAIN_VIRT_BHYVE;
    int arch = virArchFromHost(); /* virArch */

    virCheckFlags(0, ret);

    if (virConnectGetDomainCapabilitiesEnsureACL(conn) < 0)
        return ret;

    if (virttype_str &&
        (virttype = virDomainVirtTypeFromString(virttype_str)) < 0) {
        virReportError(VIR_ERR_INVALID_ARG,
                       _("unknown virttype: %1$s"),
                       virttype_str);
        goto cleanup;
    }

    if (virttype != VIR_DOMAIN_VIRT_BHYVE) {
        virReportError(VIR_ERR_INVALID_ARG,
                       _("unknown virttype: %1$s"),
                       virttype_str);
        goto cleanup;
    }

    if (arch_str && (arch = virArchFromString(arch_str)) == VIR_ARCH_NONE) {
        virReportError(VIR_ERR_INVALID_ARG,
                       _("unknown architecture: %1$s"),
                       arch_str);
        goto cleanup;
    }

    if (!ARCH_IS_X86(arch) && !ARCH_IS_ARM(arch)) {
        virReportError(VIR_ERR_NO_SUPPORT,
                       _("unsupported architecture: %1$s"),
                       virArchToString(arch));
        goto cleanup;
    }

    if (emulatorbin == NULL)
        emulatorbin = "/usr/sbin/bhyve";

    if (!(caps = virBhyveDomainCapsBuild(conn->privateData, emulatorbin,
                                         machine, arch, virttype)))
        goto cleanup;

    ret = virDomainCapsFormat(caps);

 cleanup:
    virObjectUnref(caps);
    return ret;
}

static int
bhyveDomainInterfaceStats(virDomainPtr domain,
                          const char *device,
                          virDomainInterfaceStatsPtr stats)
{
    virDomainObj *vm;
    int ret = -1;
    virDomainNetDef *net = NULL;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainInterfaceStatsEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    if (!(net = virDomainNetFind(vm->def, device)))
        goto cleanup;

    if (virNetDevTapInterfaceStats(net->ifname, stats,
                                   !virDomainNetTypeSharesHostView(net)) < 0)
        goto cleanup;

    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

#define BHYVE_SET_MEMSTAT(TAG, VAL) \
        if (i < nr_stats) { \
            stats[i].tag = TAG; \
            stats[i].val = VAL; \
            i++; \
        }

static int
bhyveDomainMemoryStats(virDomainPtr domain,
                       virDomainMemoryStatPtr stats,
                       unsigned int nr_stats,
                       unsigned int flags)
{
    virDomainObj *vm;
    unsigned maxmem;
    unsigned long long rss;
    size_t i = 0;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    if (virDomainMemoryStatsEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virProcessGetStatInfo(NULL, NULL, NULL, NULL, &rss, vm->pid, 0) < 0) {
        virReportError(VIR_ERR_OPERATION_FAILED, "%s",
                       _("cannot get RSS for domain"));
    } else {
        BHYVE_SET_MEMSTAT(VIR_DOMAIN_MEMORY_STAT_RSS, rss);
    }

    maxmem = virDomainDefGetMemoryTotal(vm->def);
    BHYVE_SET_MEMSTAT(VIR_DOMAIN_MEMORY_STAT_AVAILABLE, maxmem);

    ret = i;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

#undef BHYVE_SET_MEMSTAT

static struct kinfo_proc *bhyveDomainProcGetInfo(pid_t pid)
{
    int mib[4];
    size_t len = 4;
    struct kinfo_proc *p = g_malloc0(sizeof(struct kinfo_proc));

    sysctlnametomib("kern.proc.pid", mib, &len);
    len = sizeof(struct kinfo_proc);
    mib[3] = pid;

    if (sysctl(mib, 4, p, &len, NULL, 0) < 0) {
        virReportSystemError(errno, "%s",
                             _("Unable to query process stats"));
        return NULL;
    }

    return p;
}

static int
bhyveDomainBlockStats(virDomainPtr domain,
                      const char *path G_GNUC_UNUSED,
                      virDomainBlockStatsPtr stats)
{
    virDomainObj *vm;
    int ret = -1;
    g_autofree struct kinfo_proc *p = NULL;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    if (virDomainBlockStatsEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if ((p = bhyveDomainProcGetInfo(vm->pid)) == NULL)
        goto cleanup;

    stats->rd_req = p->ki_rusage.ru_inblock;
    stats->wr_req = p->ki_rusage.ru_oublock;
    stats->rd_bytes = -1;
    stats->wr_bytes = -1;
    stats->errs = -1;

    ret = 0;
 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetVcpusFlags(virDomainPtr domain, unsigned int flags)
{
    virDomainObj *vm;
    virDomainDef *def;
    int ret = -1;

    virCheckFlags(VIR_DOMAIN_AFFECT_LIVE |
                  VIR_DOMAIN_AFFECT_CONFIG |
                  VIR_DOMAIN_VCPU_MAXIMUM, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainGetVcpusFlagsEnsureACL(domain->conn, vm->def, flags) < 0)
        goto cleanup;

    if (!(def = virDomainObjGetOneDef(vm, flags)))
        goto cleanup;

    if (flags & VIR_DOMAIN_VCPU_MAXIMUM)
        ret = virDomainDefGetVcpusMax(def);
    else
        ret = virDomainDefGetVcpus(def);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetMaxVcpus(virDomainPtr domain)
{
    return bhyveDomainGetVcpusFlags(domain, (VIR_DOMAIN_AFFECT_LIVE |
                                             VIR_DOMAIN_VCPU_MAXIMUM));
}

static int
bhyveDomainGetVcpuPinInfo(virDomainPtr domain,
                        int ncpumaps,
                        unsigned char *cpumaps,
                        int maplen,
                        unsigned int flags)
{
    virDomainObj *vm;
    virDomainDef *def;
    g_autoptr(virBitmap) hostcpus = NULL;
    int ret = -1;

    virCheckFlags(VIR_DOMAIN_AFFECT_LIVE | VIR_DOMAIN_AFFECT_CONFIG, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainGetVcpuPinInfoEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (!(def = virDomainObjGetOneDef(vm, flags)))
        goto cleanup;

    if (!(hostcpus = virHostCPUGetAvailableCPUsBitmap()))
        goto cleanup;

    ret = virDomainDefGetVcpuPinInfoHelper(def, maplen, ncpumaps, cpumaps,
                                           hostcpus, NULL);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}


static int
bhyveDomainInterfaceAddresses(virDomainPtr domain,
                              virDomainInterfacePtr **ifaces,
                              unsigned int source,
                              unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    int ret = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainInterfaceAddressesEnsureACL(domain->conn, vm->def, source) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto cleanup;

    switch (source) {
    case VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_AGENT:
        if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
            goto cleanup;

        if (bhyveDomainEnsureAgent(vm, true) < 0)
            goto endjob;

        agent = bhyveDomainObjEnterAgent(vm);
        ret = qemuAgentGetInterfaces(agent, ifaces, true);
        bhyveDomainObjExitAgent(vm, agent);

    endjob:
        virDomainObjEndAgentJob(vm);

        break;

    case VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_ARP:
        ret = virDomainNetARPInterfaces(vm->def, ifaces);
        break;

    case VIR_DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE:
        ret = virDomainNetDHCPInterfaces(vm->def, ifaces);
        break;

    default:
        virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,
                       _("Unknown IP address data source %1$d"),
                       source);
        break;
    }

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}


static int
bhyveDomainGetHostnameAgent(virDomainObj *vm,
                            char **hostname)
{
    qemuAgent *agent;
    int ret = -1;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
        return -1;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    ret = qemuAgentGetHostname(agent, hostname, true);
    bhyveDomainObjExitAgent(vm, agent);

 endjob:
    virDomainObjEndAgentJob(vm);
    return ret;
}


static char *
bhyveDomainQemuAgentCommand(virDomainPtr domain,
                            const char *cmd,
                            int timeout,
                            unsigned int flags)
{
    virDomainObj *vm;
    int ret = -1;
    char *result = NULL;
    qemuAgent *agent;

    virCheckFlags(0, NULL);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        goto cleanup;

    if (virDomainQemuAgentCommandEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    ret = qemuAgentArbitraryCommand(agent, cmd, &result, timeout);
    bhyveDomainObjExitAgent(vm, agent);
    if (ret < 0)
        VIR_FREE(result);

 endjob:
    virDomainObjEndAgentJob(vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    return result;
}


static int
bhyveDomainGetHostnameLease(virDomainObj *vm,
                            char **hostname)
{
    char macaddr[VIR_MAC_STRING_BUFLEN];
    g_autoptr(virConnect) conn = NULL;
    virNetworkDHCPLeasePtr *leases = NULL;
    int n_leases;
    size_t i, j;
    int ret = -1;

    *hostname = NULL;

    if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0)
        return -1;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (!(conn = virGetConnectNetwork()))
        goto endjob;

    for (i = 0; i < vm->def->nnets; i++) {
        g_autoptr(virNetwork) network = NULL;
        virDomainNetDef *net = vm->def->nets[i];

        if (net->type != VIR_DOMAIN_NET_TYPE_NETWORK)
            continue;

        virMacAddrFormat(&net->mac, macaddr);
        network = virNetworkLookupByName(conn, net->data.network.name);

        if (!network)
            goto endjob;

        if ((n_leases = virNetworkGetDHCPLeases(network, macaddr,
                                                &leases, 0)) < 0)
            goto endjob;

        for (j = 0; j < n_leases; j++) {
            virNetworkDHCPLeasePtr lease = leases[j];
            if (lease->hostname && !*hostname)
                *hostname = g_strdup(lease->hostname);

            virNetworkDHCPLeaseFree(lease);
        }

        VIR_FREE(leases);

        if (*hostname)
            break;
    }

    ret = 0;
 endjob:
    virDomainObjEndJob(vm);
    return ret;
}

static char *
bhyveDomainGetHostname(virDomainPtr domain,
                       unsigned int flags)
{
    virDomainObj *vm = NULL;
    char *hostname = NULL;

    virCheckFlags(VIR_DOMAIN_GET_HOSTNAME_LEASE |
                  VIR_DOMAIN_GET_HOSTNAME_AGENT, NULL);

    VIR_EXCLUSIVE_FLAGS_RET(VIR_DOMAIN_GET_HOSTNAME_LEASE,
                            VIR_DOMAIN_GET_HOSTNAME_AGENT,
                            NULL);

    if (!(flags & VIR_DOMAIN_GET_HOSTNAME_AGENT))
        flags |= VIR_DOMAIN_GET_HOSTNAME_LEASE;

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return NULL;

    if (virDomainGetHostnameEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (flags & VIR_DOMAIN_GET_HOSTNAME_LEASE) {
        if (bhyveDomainGetHostnameLease(vm, &hostname) < 0)
            goto cleanup;
    } else if (flags & VIR_DOMAIN_GET_HOSTNAME_AGENT) {
        if (bhyveDomainGetHostnameAgent(vm, &hostname) < 0)
            goto cleanup;
    }

    if (!hostname) {
        virReportError(VIR_ERR_NO_HOSTNAME,
                       _("no hostname found for domain %1$s"),
                       vm->def->name);
        goto cleanup;
    }

 cleanup:
    virDomainObjEndAPI(&vm);
    return hostname;
}


#define BHYVE_NB_MEM_PARAM  1
#define BHYVE_ASSIGN_MEM_PARAM(index, name, value) \
    if (index < *nparams && \
        virTypedParameterAssign(&params[index], name, VIR_TYPED_PARAM_ULLONG, \
                                value) < 0) \
        goto cleanup

static int
bhyveDomainGetMemoryParameters(virDomainPtr domain,
                               virTypedParameterPtr params,
                               int *nparams,
                               unsigned int flags)
{
    virDomainObj *vm = NULL;
    virDomainDef *persistentDef = NULL;
    int ret = -1;
    unsigned long long mem_hard_limit;

    virCheckFlags(VIR_DOMAIN_AFFECT_LIVE |
                  VIR_DOMAIN_AFFECT_CONFIG |
                  VIR_TYPED_PARAM_STRING_OKAY, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainGetMemoryParametersEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjGetDefs(vm, flags, NULL, &persistentDef) < 0)
        goto cleanup;

    if ((*nparams) == 0) {
        *nparams = BHYVE_NB_MEM_PARAM;
        ret = 0;
        goto cleanup;
    }

    if (persistentDef) {
        mem_hard_limit = persistentDef->mem.hard_limit;
    } else {
        if (bhyveRctlGetMemoryHardLimit(vm->pid, &mem_hard_limit) < 0)
            goto cleanup;
    }

    BHYVE_ASSIGN_MEM_PARAM(0, VIR_DOMAIN_MEMORY_HARD_LIMIT, mem_hard_limit);

    if (BHYVE_NB_MEM_PARAM < *nparams)
        *nparams = BHYVE_NB_MEM_PARAM;
    ret = 0;

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}
#undef BHYVE_ASSIGN_MEM_PARAM

static int
bhyveDomainSetMemoryParameters(virDomainPtr domain,
                               virTypedParameterPtr params,
                               int nparams,
                               unsigned int flags)
{
    struct _bhyveConn *privconn = domain->conn->privateData;
    virDomainDef *def = NULL;
    virDomainDef *persistentDef = NULL;
    virDomainObj *vm = NULL;
    int ret = -1;
    unsigned long long hard_limit = 0;

    virCheckFlags(VIR_DOMAIN_AFFECT_LIVE |
                  VIR_DOMAIN_AFFECT_CONFIG, -1);

    if (virTypedParamsValidate(params, nparams,
                               VIR_DOMAIN_MEMORY_HARD_LIMIT,
                               VIR_TYPED_PARAM_ULLONG,
                               NULL) < 0)
        return -1;


    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainSetMemoryParametersEnsureACL(domain->conn, vm->def, flags) < 0)
        goto cleanup;

    if (virDomainObjBeginJob(vm, VIR_JOB_MODIFY) < 0)
        goto cleanup;

    if (virDomainObjGetDefs(vm, flags, &def, &persistentDef) < 0)
        goto endjob;

    if (virTypedParamsGetULLong(params, nparams, VIR_DOMAIN_MEMORY_HARD_LIMIT, &hard_limit) < 0)
        return -1;

    if (def) {
        if ((bhyveDriverGetBhyveCaps(privconn) & BHYVE_CAP_RCTL) == 0) {
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Cannot set resource limits: RACCT/RCTL is either not supported or not enabled"));
            goto endjob;
        }

        if (bhyveRctlSetMemoryHardLimit(vm->pid, hard_limit) < 0)
            goto endjob;

        def->mem.hard_limit = hard_limit;
        if (virDomainObjSave(vm, privconn->xmlopt, BHYVE_STATE_DIR) < 0)
            VIR_WARN("Failed to save status on vm %s", vm->def->name);
    }

    if (persistentDef) {
        persistentDef->mem.hard_limit = hard_limit;

        if (virDomainDefSave(persistentDef, privconn->xmlopt, BHYVE_CONFIG_DIR) < 0)
            goto endjob;
    }

    ret = 0;

 endjob:
    virDomainObjEndJob(vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetFSInfoAgent(virDomainObj *vm,
                          qemuAgentFSInfo ***info)
{
    int ret = -1;
    qemuAgent *agent;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
        return ret;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    ret = qemuAgentGetFSInfo(agent, info, true);
    bhyveDomainObjExitAgent(vm, agent);

 endjob:
    virDomainObjEndAgentJob(vm);
    return ret;
}

static int
bhyveDomainGetFSInfo(virDomainPtr dom,
                     virDomainFSInfoPtr **info,
                     unsigned int flags)
{
    virDomainObj *vm;
    qemuAgentFSInfo **agentinfo = NULL;
    int ret = -1;
    int nfs = 0;

    virCheckFlags(0, ret);

    if (!(vm = bhyveDomObjFromDomain(dom)))
        return ret;

    if (virDomainGetFSInfoEnsureACL(dom->conn, vm->def) < 0)
        goto cleanup;

    if ((nfs = bhyveDomainGetFSInfoAgent(vm, &agentinfo)) < 0)
        goto cleanup;

    if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    ret = qemuAgentFSInfoFormat(agentinfo, nfs, vm->def, info);

 endjob:
    virDomainObjEndJob(vm);

 cleanup:
    if (agentinfo) {
        size_t i;
        for (i = 0; i < nfs; i++)
            qemuAgentFSInfoFree(agentinfo[i]);
        g_free(agentinfo);
    }
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainGetTime(virDomainPtr domain,
                   long long *seconds,
                   unsigned int *nseconds,
                   unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    int ret = -1;
    int rv;

    virCheckFlags(0, ret);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return ret;

    if (virDomainGetTimeEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    rv = qemuAgentGetTime(agent, seconds, nseconds);
    bhyveDomainObjExitAgent(vm, agent);

    if (rv < 0)
        goto endjob;

    ret = 0;

 endjob:
    virDomainObjEndAgentJob(vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainSetTime(virDomainPtr domain,
                   long long seconds,
                   unsigned int nseconds,
                   unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    bool rtcSync = flags & VIR_DOMAIN_TIME_SYNC;
    int ret = -1;
    int rv;

    virCheckFlags(VIR_DOMAIN_TIME_SYNC, ret);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return ret;

    if (virDomainSetTimeEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    rv = qemuAgentSetTime(agent, seconds, nseconds, rtcSync);
    bhyveDomainObjExitAgent(vm, agent);

    if (rv < 0)
        goto endjob;

    ret = 0;

 endjob:
    virDomainObjEndAgentJob(vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainSetUserPassword(virDomainPtr domain,
                           const char *user,
                           const char *password,
                           unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    int ret = -1;
    int rv;

    virCheckFlags(VIR_DOMAIN_PASSWORD_ENCRYPTED, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return ret;

    if (virDomainSetUserPasswordEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    rv = qemuAgentSetUserPassword(agent, user, password,
                                  flags & VIR_DOMAIN_PASSWORD_ENCRYPTED);
    bhyveDomainObjExitAgent(vm, agent);

    if (rv < 0)
        goto endjob;

    ret = 0;

 endjob:
    virDomainObjEndAgentJob(vm);

 cleanup:
    virDomainObjEndAPI(&vm);
    return ret;
}

static int
bhyveDomainAuthorizedSSHKeysGet(virDomainPtr domain,
                                const char *user,
                                char ***keys,
                                unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    int rv = -1;

    virCheckFlags(0, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainAuthorizedSshKeysGetEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_QUERY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    rv = qemuAgentSSHGetAuthorizedKeys(agent, user, keys);
    bhyveDomainObjExitAgent(vm, agent);

 endjob:
    virDomainObjEndAgentJob(vm);
 cleanup:
    virDomainObjEndAPI(&vm);
    return rv;
}

static int
bhyveDomainAuthorizedSSHKeysSet(virDomainPtr domain,
                                const char *user,
                                const char **keys,
                                unsigned int nkeys,
                                unsigned int flags)
{
    virDomainObj *vm = NULL;
    qemuAgent *agent;
    const bool append = flags & VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_APPEND;
    const bool remove = flags & VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_REMOVE;
    int rv = -1;

    virCheckFlags(VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_APPEND |
                  VIR_DOMAIN_AUTHORIZED_SSH_KEYS_SET_REMOVE, -1);

    if (!(vm = bhyveDomObjFromDomain(domain)))
        return -1;

    if (virDomainAuthorizedSshKeysSetEnsureACL(domain->conn, vm->def) < 0)
        goto cleanup;

    if (virDomainObjBeginAgentJob(vm, VIR_AGENT_JOB_MODIFY) < 0)
        goto cleanup;

    if (virDomainObjCheckActive(vm) < 0)
        goto endjob;

    if (bhyveDomainEnsureAgent(vm, true) < 0)
        goto endjob;

    agent = bhyveDomainObjEnterAgent(vm);
    if (remove)
        rv = qemuAgentSSHRemoveAuthorizedKeys(agent, user, keys, nkeys);
    else
        rv = qemuAgentSSHAddAuthorizedKeys(agent, user, keys, nkeys, !append);
    bhyveDomainObjExitAgent(vm, agent);

 endjob:
    virDomainObjEndAgentJob(vm);
 cleanup:
    virDomainObjEndAPI(&vm);
    return rv;
}

static virHypervisorDriver bhyveHypervisorDriver = {
    .name = "bhyve",
    .connectURIProbe = bhyveConnectURIProbe,
    .connectOpen = bhyveConnectOpen, /* 1.2.2 */
    .connectClose = bhyveConnectClose, /* 1.2.2 */
    .connectGetVersion = bhyveConnectGetVersion, /* 1.2.2 */
    .connectGetHostname = bhyveConnectGetHostname, /* 1.2.2 */
    .connectGetSysinfo = bhyveConnectGetSysinfo, /* 1.2.5 */
    .domainGetInfo = bhyveDomainGetInfo, /* 1.2.2 */
    .domainGetState = bhyveDomainGetState, /* 1.2.2 */
    .connectGetCapabilities = bhyveConnectGetCapabilities, /* 1.2.2 */
    .connectListDomains = bhyveConnectListDomains, /* 1.2.2 */
    .connectNumOfDomains = bhyveConnectNumOfDomains, /* 1.2.2 */
    .connectListAllDomains = bhyveConnectListAllDomains, /* 1.2.2 */
    .connectListDefinedDomains = bhyveConnectListDefinedDomains, /* 1.2.2 */
    .connectNumOfDefinedDomains = bhyveConnectNumOfDefinedDomains, /* 1.2.2 */
    .connectDomainXMLToNative = bhyveConnectDomainXMLToNative, /* 1.2.5 */
    .domainCreate = bhyveDomainCreate, /* 1.2.2 */
    .domainCreateWithFlags = bhyveDomainCreateWithFlags, /* 1.2.3 */
    .domainCreateXML = bhyveDomainCreateXML, /* 1.2.4 */
    .domainDestroy = bhyveDomainDestroy, /* 1.2.2 */
    .domainDestroyFlags = bhyveDomainDestroyFlags, /* 5.6.0 */
    .domainShutdown = bhyveDomainShutdown, /* 1.3.3 */
    .domainShutdownFlags = bhyveDomainShutdownFlags, /* 5.6.0 */
    .domainReboot = bhyveDomainReboot, /* TBD */
    .domainLookupByUUID = bhyveDomainLookupByUUID, /* 1.2.2 */
    .domainLookupByName = bhyveDomainLookupByName, /* 1.2.2 */
    .domainLookupByID = bhyveDomainLookupByID, /* 1.2.3 */
    .domainDefineXML = bhyveDomainDefineXML, /* 1.2.2 */
    .domainDefineXMLFlags = bhyveDomainDefineXMLFlags, /* 1.2.12 */
    .domainUndefine = bhyveDomainUndefine, /* 1.2.2 */
    .domainUndefineFlags = bhyveDomainUndefineFlags, /* 5.6.0 */
    .domainGetOSType = bhyveDomainGetOSType, /* 1.2.21 */
    .domainGetXMLDesc = bhyveDomainGetXMLDesc, /* 1.2.2 */
    .domainIsActive = bhyveDomainIsActive, /* 1.2.2 */
    .domainIsPersistent = bhyveDomainIsPersistent, /* 1.2.2 */
    .domainGetAutostart = bhyveDomainGetAutostart, /* 1.2.4 */
    .domainSetAutostart = bhyveDomainSetAutostart, /* 1.2.4 */
    .domainOpenConsole = bhyveDomainOpenConsole, /* 1.2.4 */
    .domainSetMetadata = bhyveDomainSetMetadata, /* 1.2.4 */
    .domainGetMetadata = bhyveDomainGetMetadata, /* 1.2.4 */
    .nodeGetCPUStats = bhyveNodeGetCPUStats, /* 1.2.2 */
    .nodeGetMemoryStats = bhyveNodeGetMemoryStats, /* 1.2.2 */
    .nodeGetInfo = bhyveNodeGetInfo, /* 1.2.3 */
    .connectGetMaxVcpus = bhyveConnectGetMaxVcpus, /* 1.2.3 */
    .nodeGetFreeMemory = bhyveNodeGetFreeMemory, /* 1.2.3 */
    .nodeGetCPUMap = bhyveNodeGetCPUMap, /* 1.2.3 */
    .nodeGetMemoryParameters = bhyveNodeGetMemoryParameters, /* 1.2.3 */
    .nodeSetMemoryParameters = bhyveNodeSetMemoryParameters, /* 1.2.3 */
    .connectBaselineCPU = bhyveConnectBaselineCPU, /* 1.2.4 */
    .connectCompareCPU = bhyveConnectCompareCPU, /* 1.2.4 */
    .connectDomainEventRegisterAny = bhyveConnectDomainEventRegisterAny, /* 1.2.5 */
    .connectDomainEventDeregisterAny = bhyveConnectDomainEventDeregisterAny, /* 1.2.5 */
    .domainHasManagedSaveImage = bhyveDomainHasManagedSaveImage, /* 1.2.13 */
    .connectGetType = bhyveConnectGetType, /* 1.3.5 */
    .connectIsAlive = bhyveConnectIsAlive, /* 1.3.5 */
    .connectIsSecure = bhyveConnectIsSecure, /* 1.3.5 */
    .connectIsEncrypted = bhyveConnectIsEncrypted, /* 1.3.5 */
    .connectDomainXMLFromNative = bhyveConnectDomainXMLFromNative, /* 2.1.0 */
    .connectGetDomainCapabilities = bhyveConnectGetDomainCapabilities, /* 2.1.0 */
    .domainInterfaceStats = bhyveDomainInterfaceStats, /* 11.7.0 */
    .domainMemoryStats = bhyveDomainMemoryStats, /* 11.7.0 */
    .domainBlockStats = bhyveDomainBlockStats, /* 11.7.0 */
    .domainGetVcpusFlags = bhyveDomainGetVcpusFlags, /* 12.1.0 */
    .domainGetMaxVcpus = bhyveDomainGetMaxVcpus, /* 12.1.0 */
    .domainGetVcpuPinInfo = bhyveDomainGetVcpuPinInfo, /* 12.1.0 */
    .domainInterfaceAddresses = bhyveDomainInterfaceAddresses, /* 12.3.0 */
    .domainGetHostname = bhyveDomainGetHostname, /* 12.3.0 */
    .domainQemuAgentCommand = bhyveDomainQemuAgentCommand, /* 12.4.0 */
    .domainGetMemoryParameters = bhyveDomainGetMemoryParameters, /* 12.4.0 */
    .domainSetMemoryParameters = bhyveDomainSetMemoryParameters, /* 12.4.0 */
    .domainGetFSInfo = bhyveDomainGetFSInfo, /* 12.5.0 */
    .domainGetTime = bhyveDomainGetTime, /* 12.6.0 */
    .domainSetTime = bhyveDomainSetTime, /* 12.6.0 */
    .domainSetUserPassword = bhyveDomainSetUserPassword, /* 12.6.0 */
    .domainAuthorizedSSHKeysGet = bhyveDomainAuthorizedSSHKeysGet, /* 12.6.0 */
    .domainAuthorizedSSHKeysSet = bhyveDomainAuthorizedSSHKeysSet, /* 12.6.0 */
};


static virConnectDriver bhyveConnectDriver = {
    .localOnly = true,
    .uriSchemes = (const char *[]){ "bhyve", NULL },
    .hypervisorDriver = &bhyveHypervisorDriver,
};

static virStateDriver bhyveStateDriver = {
    .name = "bhyve",
    .stateInitialize = bhyveStateInitialize,
    .stateCleanup = bhyveStateCleanup,
};

int
bhyveRegister(void)
{
    if (virRegisterConnectDriver(&bhyveConnectDriver,
                                 true) < 0)
        return -1;
    if (virRegisterStateDriver(&bhyveStateDriver) < 0)
        return -1;
    return 0;
}
