VPP vpp_main線程時間輪用法

vpp_main線程經過在文件vppsrcvlibmain.c中包含#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>指定了使用1t_3w_1024sl_ov類型時間輪構建定時器。

數據結構

• [ ] vlib_node_main_t

/* 線程與定時器相關成員 */
typedef struct
{
    ......
    
    /* Timing wheel for scheduling time-based node dispatch. */
    /* 基於時間的調度策略，依賴時間輪構建，使用的是1t_3w_1024sl_ov類型時間輪 */
    void *timing_wheel;
    /* vpp_main線程註冊的定時器的私有數據保存向量，索引來自data_from_advancing_timing_wheel */
    vlib_signal_timed_event_data_t *signal_timed_event_data_pool;

    /* Opaque data vector added via timing_wheel_advance. */
    /* 已經發生的定時事件的用戶句柄保存內存 */
    u32 *data_from_advancing_timing_wheel;

    /* CPU time of next process to be ready on timing wheel. */
    f64 time_next_process_ready;/* 該值目前沒有使用，根據意思是最近一個可能超時的時間 */
    ......

} vlib_node_main_t;

• [ ] vlib_signal_timed_event_data_t

/* 定時器私有數據 */
typedef struct
{
    u16 n_data_elts;/* 數據元素個數 */

    u16 n_data_elt_bytes;/* 每一個元素字節數 */

    /* n_data_elts * n_data_elt_bytes 即下面union存放的數據大小*/
    u32 n_data_bytes;

    /* Process node & event type to be used to signal event. */
    /* 該事件所屬協程 */
    u32 process_node_index;

    u32 event_type_index;/* 事件類型索引，用於通知協程的事件通知機制什麼事情發生了 */
    /* 私有數據存放位置，數據較少時，放在靜態數組inline_event_data中，
    ** 不然放在動態內存event_data_as_vector中 */
    union
    {
        u8 inline_event_data[64 - 3 * sizeof (u32) - 2 * sizeof (u16)];

        /* Vector of event data used only when data does not fit inline. */
        u8 *event_data_as_vector;
    };
}vlib_signal_timed_event_data_t;

代碼流程

初始化時間輪

/* Main function. */
int
vlib_main (vlib_main_t * volatile vm, unformat_input_t * input)
{
    ......
    /* 分配時間輪描述控制塊 */
    nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
                       CLIB_CACHE_LINE_BYTES);
    /* 確保能夠存放10個事件 */
    vec_validate (nm->data_from_advancing_timing_wheel, 10);
    _vec_len (nm->data_from_advancing_timing_wheel) = 0;

    /* Create the process timing wheel */
    /* 建立時間輪 */
    TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
                              0 /* no callback */ ,
                              10e-6 /* timer period 10us */ ,
                              ~0 /* max expirations per call */ );

    ......

    return 0;
}

定時器用戶ID構成原則

定時器的用戶ID由兩部分構成，私有數據索引和類型標誌位，最低位爲類型標誌位：

• 1表示時間類定時器，
• 0表示協程掛起類定時器(這類定時器給協程使用，協程使用該類型定時器掛起一段時間)。

/* 判斷是否爲時間類定時器，直接獲取最低位值 */
always_inline uword
vlib_timing_wheel_data_is_timed_event (u32 d)
{
    return d & 1;
}
/* 構建協程掛起事件定時器用戶id */
always_inline u32
vlib_timing_wheel_data_set_suspended_process (u32 i)
{
    return 0 + 2 * i;
}
/* 構建時間類定時器用戶id */
always_inline u32
vlib_timing_wheel_data_set_timed_event (u32 i)
{
    return 1 + 2 * i;
}
/* 獲取私有數據索引 */
always_inline uword
vlib_timing_wheel_data_get_index (u32 d)
{
    return d / 2;
}

VPP_MAIN線程驅動時間輪

/* 參數is_main決定是主線程仍是worker線程 */
static_always_inline void
vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
{
    ......
    
    while (1)
    {
        ......
        
        if (is_main) /* 只有主協程纔會調度該時間輪 */
        {
            /* *INDENT-OFF* */
            ELOG_TYPE_DECLARE (es) =
            {
                .format = "process tw start",
                .format_args = "",
            };
            ELOG_TYPE_DECLARE (ee) =
            {
                .format = "process tw end: %d",
                .format_args = "i4",
            };
            /* *INDENT-ON* */

            struct
            {
                int nready_procs;
            }*ed;

            /* Check if process nodes have expired from timing wheel. */
            ASSERT (nm->data_from_advancing_timing_wheel != 0);

            if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
                ed = ELOG_DATA (&vlib_global_main.elog_main, es);
            /* 執行定時器超時操做，查看是否有定時器超時，處理超時事件，超時句柄保存在                                            data_from_advancing_timing_wheel中 */
            nm->data_from_advancing_timing_wheel =
                TW (tw_timer_expire_timers_vec)
                ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
                 nm->data_from_advancing_timing_wheel);

            ASSERT (nm->data_from_advancing_timing_wheel != 0);

            if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
            {
                ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
                ed->nready_procs =
                    _vec_len (nm->data_from_advancing_timing_wheel);
            }
            /* 存在超時事件 */
            if (PREDICT_FALSE(_vec_len (nm->data_from_advancing_timing_wheel) > 0))
            {
                uword i;
                /* 遍歷超時事件 */
                for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel); i++)
                {
                    u32 d = nm->data_from_advancing_timing_wheel[i];
                    /* 句柄轉換爲索引，由於最低爲爲類型位，因此直接除以二 */
                    u32 di = vlib_timing_wheel_data_get_index (d);

                    if (vlib_timing_wheel_data_is_timed_event (d))/* 最低位爲1說明是通常定時事件 */
                    {
                        vlib_signal_timed_event_data_t *te =
                            pool_elt_at_index (nm->signal_timed_event_data_pool, di);
                        vlib_node_t *n = vlib_get_node (vm, te->process_node_index);
                        vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
                        void *data;
                        /* 通知事件處理,通知指定協程，定時事件發生了 */
                        data = vlib_process_signal_event_helper (nm, n, p,
                                te->event_type_index,
                                te->n_data_elts,
                                te->n_data_elt_bytes);
                        /* 透明數據拷貝 */
                        if (te->n_data_bytes < sizeof (te->inline_event_data))
                            clib_memcpy_fast (data, te->inline_event_data, te->n_data_bytes);
                        else
                        {
                            clib_memcpy_fast (data, te->event_data_as_vector,
                                              te->n_data_bytes);
                            vec_free (te->event_data_as_vector);
                        }
                        /* 將te從signal_timed_event_data_pool移除 */
                        pool_put (nm->signal_timed_event_data_pool, te);
                    }
                    else /* 協程類定時事件 */
                    {
                        /* 處理協程定時事件，即喚醒協程 */
                        cpu_time_now = clib_cpu_time_now ();
                        cpu_time_now = dispatch_suspended_process (vm, di, cpu_time_now);
                    }
                }
                _vec_len (nm->data_from_advancing_timing_wheel) = 0;
            }
        }
        vlib_increment_main_loop_counter (vm);

        /* Record time stamp in case there are no enabled nodes and above
            calls do not update time stamp. */
        cpu_time_now = clib_cpu_time_now ();
    }
}

協程類定時器註冊

static u64
dispatch_process (vlib_main_t * vm,
                  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
{
    ......
    
    if (is_suspend)
    {
        vlib_pending_frame_t *pf;

        n_vectors = 0;
        pool_get (nm->suspended_process_frames, pf);
        pf->node_runtime_index = node->runtime_index;
        pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
        pf->next_frame_index = ~0;

        p->n_suspends += 1;/* 統計掛起數量 */
        p->suspended_process_frame_index = pf - nm->suspended_process_frames;
        /* 設置了使用定時器掛起標誌，則經過定時器掛起該線程 */
        if (p->flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK)
        {
            TWT (tw_timer_wheel) * tw =
                (TWT (tw_timer_wheel) *) nm->timing_wheel;
            p->stop_timer_handle =
                TW (tw_timer_start) (tw,
                                     vlib_timing_wheel_data_set_suspended_process
                                     (node->runtime_index) /* [sic] pool idex */ ,
                                     0 /* timer_id */ ,
                                     p->resume_clock_interval);
        }
    }
    else
        p->flags &= ~VLIB_PROCESS_IS_RUNNING;

    t = clib_cpu_time_now ();

    vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
                               /* is_after */ 1);

    vlib_process_update_stats (vm, p,
                               /* n_calls */ !is_suspend,
                               /* n_vectors */ n_vectors,
                               /* n_clocks */ t - last_time_stamp);

    return t;
}

普通定時器註冊

以quic.c模塊使用了vpp_main線程的定時器爲例

static void
quic_update_timer (quic_ctx_t * ctx)
{
    tw_timer_wheel_1t_3w_1024sl_ov_t *tw;
    int64_t next_timeout;

    // This timeout is in ms which is the unit of our timer
    next_timeout = quicly_get_first_timeout (ctx->c_quic_ctx_id.conn);
    tw = &quic_main.wrk_ctx[vlib_get_thread_index ()].timer_wheel;
    f64 next_timeout_f = ((f64) next_timeout) / 1000.f;

    clib_warning ("Timer set to %ld (%lf)", next_timeout, next_timeout_f);

    if (ctx->c_quic_ctx_id.timer_handle == QUIC_TIMER_HANDLE_INVALID)
    {
        if (next_timeout == INT64_MAX)
            return;
        ctx->c_quic_ctx_id.timer_handle =
            tw_timer_start_1t_3w_1024sl_ov (tw, ctx->c_c_index, 0,
                                            next_timeout_f);
    }
    else
    {
        if (next_timeout == INT64_MAX)
        {
            tw_timer_stop_1t_3w_1024sl_ov (tw, ctx->c_quic_ctx_id.timer_handle);
            ctx->c_quic_ctx_id.timer_handle = QUIC_TIMER_HANDLE_INVALID;
        }
        else
            tw_timer_update_1t_3w_1024sl_ov (tw, ctx->c_quic_ctx_id.timer_handle,
                                             next_timeout_f);
    }
}