libvirt API非阻塞調用及相關的原理分析

如下的分析基於libvirt 3.0版本。

   libvirt是一套免費，開源的支持linux下的主流虛擬化管理工具，目前有大量的應用程序構建在libvirt之上，不少虛擬化產品的開發都是靈活調用libvirt的API接口去實現的。對於應用程序，libvirt提供一套非阻塞調用的框架。

   涉及相關的API：

   virInitialize：初始化libvirt庫，主要針對多線程編程

   virEventRegisterDefaultImpl：基於poll系統調用註冊默認事件實現，這是一個通用實現。

   virEventRunDefaultImpl：循環運行事件，須要在線程中單獨運行該函數

   virConnectOpen：鏈接libvirt服務端，即libvirtd

   virConnectSetKeepAlive：設置保活週期，此函數控制客戶端發送keepalive消息。

相關的實現代碼以下：

if (virInitialize()< 0) {

          printf("callvirInitialize initialize libvirt fail\n");

     return 1;

}

   

/* register defaultlibvirt event implement */

if (virEventRegisterDefaultImpl() < 0) { //必須

    printf("failed to registerdefault event implementation\n");

    return 2;

}

 

//建立線程去分發事件，必須

void*libvirt_thread_cb(void *data)

{

    (void)data;

   

    while (!isexit) {

        virEventRunDefaultImpl();

    }

    pthread_exit((void*)"libvirt pthreadwill exit!!!");

    return (void*)0;

}

 

//鏈接libvirt服務端，而且設計保活機制

contor =virConnectOpen(NULL);//參數爲空，表示鏈接本地的libvirtd服務端

if (contor) {

    //第二個參數，心跳發送週期；  第三個參數，心跳參數，當超過該次數時，鏈接斷開

    if (virConnectSetKeepAlive(contor, 10, 6)< 0) {

       printf("failed to set connkeep alive config\n");

        virConnectClose(contor);

        contor = NULL;

    }

}

經過以上的實現，後續就能夠非阻塞調用libvirt其它的API接口，當libvirtd阻塞時，可以超時返回。

對於libvirtd端相關的心跳週期保存在libvirtd.conf文件中，能夠修改參數，而後再重啓libvirtd便可生效：

keepalive_interval = 10 //default is 5s

keepalive_count = 6 //default is 5s

相關的源碼分析：

virEventRegisterDefaultImpl函數分析，源碼以下：

int virEventRegisterDefaultImpl(void)
{
    VIR_DEBUG("registering default event implementation");

    virResetLastError();

    if (virEventPollInit() < 0) {
        virDispatchError(NULL);
        return -1;
    }

    virEventRegisterImpl(
        virEventPollAddHandle,
        virEventPollUpdateHandle,
        virEventPollRemoveHandle,
        virEventPollAddTimeout,
        virEventPollUpdateTimeout,
        virEventPollRemoveTimeout
        );

    return 0;
}

經過源碼分析，調用virEventRegisterImpl函數對全局函數賦值，爲何會這樣作，下一步再分析

virConnectSetKeepAlive函數分析：

經過源碼分析，最終會調用virKeepAliveStart函數去啓用定時器，發送keepalive消息。

virKeepAliveStart(virKeepAlivePtr ka,
                  int interval,
                  unsigned int count)
{
    int ret = -1;
    time_t delay;
    int timeout;
    time_t now;

    virObjectLock(ka);

    if (ka->timer >= 0) {//若是定時器存在，不作處理
        VIR_DEBUG("Keepalive messages already enabled");
        ret = 0;
        goto cleanup;
    }

    if (interval > 0) {
        if (ka->interval > 0) {//心跳週期已設置，不在設置
            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                           _("keepalive interval already set"));
            goto cleanup;
        }
        /* Guard against overflow */
        if (interval > INT_MAX / 1000) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("keepalive interval %d too large"), interval);
            goto cleanup;
        }
        ka->interval = interval;
        ka->count = count;
        ka->countToDeath = count;
    }

    if (ka->interval <= 0) {//心跳週期小於0，禁用keepalive
        VIR_DEBUG("Keepalive messages disabled by configuration");
        ret = 0;
        goto cleanup;
    }

    PROBE(RPC_KEEPALIVE_START,
          "ka=%p client=%p interval=%d count=%u",
          ka, ka->client, interval, count);

    now = time(NULL);
    delay = now - ka->lastPacketReceived;
    if (delay > ka->interval)
        timeout = 0;
    else
        timeout = ka->interval - delay;
    ka->intervalStart = now - (ka->interval - timeout);
    ka->timer = virEventAddTimeout(timeout * 1000, virKeepAliveTimer,
                                   ka, virObjectFreeCallback);//建立心跳定時器
    if (ka->timer < 0)
        goto cleanup;

    /* the timer now has another reference to this object */
    virObjectRef(ka);
    ret = 0;

 cleanup:
    virObjectUnlock(ka);
    return ret;
}

繼續分析virEventAddTimeout函數，源碼以下：

int
virEventAddTimeout(int timeout,
                   virEventTimeoutCallback cb,
                   void *opaque,
                   virFreeCallback ff)
{
    if (!addTimeoutImpl)
        return -1;

    return addTimeoutImpl(timeout, cb, opaque, ff);
}

該函數實現很簡單，調用全局的函數去設置定時器，addTimeoutImpl是全局的函數接口，這個函數的賦值是應用程式調用virEventRegisterDefaultImpl函數去設置的，這是爲何須要調用virEventRegisterDefaultImpl函數的緣由。

若是沒有提供以上的設置，爲何會阻塞，阻塞在哪一個地方，經過gdb的調用，發現阻塞在virNetClientIOEventLoop函數中的poll系統調用上。堆棧以下：

(gdb) bt
#0  virNetClientIOEventLoop (client=0x2026d30, thiscall=0x1dc57a0) at ../../../src/rpc/virnetclient.c:1595
#1  0x00007fd5dc8368a3 in virNetClientIO (client=0x2026d30, thiscall=0x1dc57a0) at ../../../src/rpc/virnetclient.c:1950
#2  0x00007fd5dc8370b7 in virNetClientSendInternal (client=0x2026d30, msg=0x2026c60, expectReply=true, nonBlock=false) at ../../../src/rpc/virnetclient.c:2122
#3  0x00007fd5dc837141 in virNetClientSendWithReply (client=0x2026d30, msg=0x2026c60) at ../../../src/rpc/virnetclient.c:2150
#4  0x00007fd5dc838048 in virNetClientProgramCall (prog=0x2027150, client=0x2026d30, serial=8, proc=212, noutfds=0, outfds=0x0, ninfds=0x0, infds=0x0, args_filter=0x7fd5dc82ce9f <xdr_remote_domain_get_state_args>,
    args=0x7fffd92ffba0, ret_filter=0x7fd5dc82cf19 <xdr_remote_domain_get_state_ret>, ret=0x7fffd92ffb80) at ../../../src/rpc/virnetclientprogram.c:329
#5  0x00007fd5dc819442 in callFull (conn=0x1a24380, priv=0x1a77660, flags=0, fdin=0x0, fdinlen=0, fdout=0x0, fdoutlen=0x0, proc_nr=212, args_filter=0x7fd5dc82ce9f <xdr_remote_domain_get_state_args>,
    args=0x7fffd92ffba0 "0T\002\002", ret_filter=0x7fd5dc82cf19 <xdr_remote_domain_get_state_ret>, ret=0x7fffd92ffb80 "") at ../../../src/remote/remote_driver.c:6637
#6  0x00007fd5dc819515 in call (conn=0x1a24380, priv=0x1a77660, flags=0, proc_nr=212, args_filter=0x7fd5dc82ce9f <xdr_remote_domain_get_state_args>, args=0x7fffd92ffba0 "0T\002\002",
    ret_filter=0x7fd5dc82cf19 <xdr_remote_domain_get_state_ret>, ret=0x7fffd92ffb80 "") at ../../../src/remote/remote_driver.c:6659
#7  0x00007fd5dc7fd77b in remoteDomainGetState (domain=0x1dc8f60, state=0x7fffd92ffcdc, reason=0x0, flags=0) at ../../../src/remote/remote_driver.c:2458
#8  0x00007fd5dc7b5b2f in virDomainGetState (domain=0x1dc8f60, state=0x7fffd92ffcdc, reason=0x0, flags=0) at ../../../src/libvirt-domain.c:2495

virNetClientIOEventLoop函數分析：

static int virNetClientIOEventLoop(virNetClientPtr client,
                                   virNetClientCallPtr thiscall)
{
    struct pollfd fds[2];
    int ret;

    fds[0].fd = virNetSocketGetFD(client->sock);
    fds[1].fd = client->wakeupReadFD;

    for (;;) {
        /* If we are non-blocking, then we don't want to sleep in poll() */
        if (thiscall->nonBlock)
            timeout = 0;

        /* Limit timeout so that we can send keepalive request in time */
        if (timeout == -1)
            timeout = virKeepAliveTimeout(client->keepalive);//返回-1，致使poll阻塞

        fds[0].events = fds[0].revents = 0;
        fds[1].events = fds[1].revents = 0;

        fds[1].events = POLLIN;

        /* Calculate poll events for calls */
        virNetClientCallMatchPredicate(client->waitDispatch,
                                       virNetClientIOEventLoopPollEvents,
                                       &fds[0]);

        if (client->nstreams)
            fds[0].events |= POLLIN;

    repoll:
        ret = poll(fds, ARRAY_CARDINALITY(fds), timeout);
        if (ret < 0 && (errno == EAGAIN || errno == EINTR))
            goto repoll;

}

        經過源碼能夠分析，因爲應用程序沒有調用virConnectSetKeepAlive函數設置心跳保活機制，致使client->keepalive爲NULL，分析virKeepAliveTimeout函數可知，當client->keepalive爲NULL時，直接返回爲-1；致使poll系統調用一直阻塞，直到有事件響應。

       總結：經過以上設置，調用virConnectOpen函數鏈接libvirt的時，就能夠實現非阻塞調用libvirt其它的API，當libvirt阻塞時，不會致使調用者阻塞。

       當libvirt主線程阻塞時，上述的設置並不能解決virConnectOpen阻塞的問題，須要修改libvirt相關的代碼。至於爲何，本身去思考，怎麼解決這個問題？