一個linux 4.9，4.14內核的bbr帶寬估計偏低問題

時間 2019-11-13

標籤一個 linux 4.9 4.14 內核 bbr 帶寬估計偏低問題欄目 Linux 简体版

原文原文鏈接

linux 4.9內核，bbr的帶寬估計問題。html

一個正常的bbr流量圖：linux

對應的ttl圖形:git

一個異常的bbr流量圖：服務器

能夠看出，異常的bbr流量圖，出現了一個很低的帶寬，且穩定在這個帶寬10s左右，而正常狀況下，這個文件下載不該該超過10s，因爲流量消耗大於流量的下載，致使了用戶播放卡頓。app

經過分析，咱們確認了bbr在應對delay_ack時，出現帶寬估計偏低的狀況，好比正常的min_rtt是1.3ms，可是delay_ack的時候，是40ms左右，而因爲probe_rtt須要10秒以後進行，那麼對應的tcp

bw乘以最小的rtt則處於偏低的狀態，在穩定10s以後，bbr會主動探測rtt，ide

bbr探測rtt的時候，行爲是什麼樣的呢？對於4.9內核版原本說，是最多發送4個mss包：oop

 cwnd = max(cwnd_gain * bottleneck_bandwidth * min_rtt, 4)

/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode: */

static const u32 bbr_probe_rtt_mode_ms = 200;

從10.3s左右的時間的流量圖來看，能夠看到一個很明顯的向下缺口，就是探測最小rtt的行爲，每一個rtt以後4個包，流量再次降低。測試

這個持續的時間爲0.2s，也就是代碼裏面以下的描述：ui

/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and
 * periodically drain the bottleneck queue, to converge to measure the true
 * min_rtt (unloaded propagation delay). This allows the flows to keep queues
 * small (reducing queuing delay and packet loss) and achieve fairness among
 * BBR flows.
 *
 * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires,
 * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets.
 * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed
 * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and
 * re-enter the previous mode. BBR uses 200ms to approximately bound the
 * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s).
 *
 * Note that flows need only pay 2% if they are busy sending over the last 10
 * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have
 * natural silences or low-rate periods within 10 seconds where the rate is low
 * enough for long enough to drain its queue in the bottleneck. We pick up
 * these min RTT measurements opportunistically with our min_rtt filter. :-)
 */
static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs)

雖然通過10s，根據探測rtt以後的min_rtt應該修正爲40ms左右，會致使流量上升，但因爲咱們的分片文件已經播放卡頓超時，客戶端會斷鏈。

結論：若是下載的流媒體文件太小，不建議開啓bbr。特別是收包的客戶端進入delay_ack很是早，而且不怎麼出現亂序和丟包的狀況。此現象在4.14內核依然存在。

回到問題自己，問什麼會出現帶寬估計太低的行爲，咱們經過什麼樣的手段去復現這種行爲，因爲現場的機頂盒咱們無法拿到，那麼只能經過本身來模擬：

首先，構建一個獲取文件的請求，http，而後被動接收發包就行，結果復現不了，也就是客戶端這邊的delay_ack並不會在bbr的startup階段出現。

因此，咱們反過來想，咱們能不能讓服務器端收到的ack進行彙集，在startup階段就收到彙集ack呢，順着這種思路，咱們仿照gro，將收到的ack進行彙集，並在40ms左右將ack報文送上協議棧

，這樣從服務器的角度，讓他認爲客戶端的ack就是這種頻率發回來。

而後，咱們在服務器bbr相關代碼地方加printk：

首先增長四個地方，就是bbr的狀態切換的地方：

static void bbr_reset_startup_mode(struct sock *sk)
{
        struct bbr *bbr = inet_csk_ca(sk);

        bbr->mode = BBR_STARTUP;
        if(((struct inet_sock*)sk)->inet_dport == 3000)
        {
        printk("mode 2 mode=%d,min_rtt_us=%d,full_bw=%d,cycle_idx=%d,pacing_gain=%d,cwnd_gain=%d,rtt_cnt=%d\r\n",bbr->mode,bbr->min_rtt_us,bbr->full_bw,bbr->cycle_idx,bbr->pacing_gain,bbr->cwnd_gain,bbr->rtt_cnt);
        }
       
。。。。
}



static void bbr_reset_probe_bw_mode(struct sock *sk)
{
        struct bbr *bbr = inet_csk_ca(sk);

        bbr->mode = BBR_PROBE_BW;
        if(((struct inet_sock*)sk)->inet_dport == 3000)
        {
        printk("mode 2 mode=%d,min_rtt_us=%d,full_bw=%d,cycle_idx=%d,pacing_gain=%d,cwnd_gain=%d,rtt_cnt=%d\r\n",bbr->mode,bbr->min_rtt_us,bbr->full_bw,bbr->cycle_idx,bbr->pacing_gain,bbr->cwnd_gain,bbr->rtt_cnt);
        }
。。。。
}

static void bbr_check_drain(struct sock *sk, const struct rate_sample *rs)
{
        struct bbr *bbr = inet_csk_ca(sk);

        if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {
                bbr->mode = BBR_DRAIN;  /* drain queue we created */
        if(((struct inet_sock*)sk)->inet_dport == 3000)
        {
        printk("mode 2 mode=%d,min_rtt_us=%d,full_bw=%d,cycle_idx=%d,pacing_gain=%d,cwnd_gain=%d,rtt_cnt=%d\r\n",bbr->mode,bbr->min_rtt_us,bbr->full_bw,bbr->cycle_idx,bbr->pacing_gain,bbr->cwnd_gain,bbr->rtt_cnt);
        }

。。。
}


static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs)
{
        struct tcp_sock *tp = tcp_sk(sk);
        struct bbr *bbr = inet_csk_ca(sk);
        bool filter_expired;
        static int count_expire=0;

  。。。。。。if (rs->rtt_us >= 0 &&
            (rs->rtt_us <= bbr->min_rtt_us || filter_expired)) {
                bbr->min_rtt_us = rs->rtt_us;
                bbr->min_rtt_stamp = tcp_time_stamp;
        }

        if (bbr_probe_rtt_mode_ms > 0 && filter_expired &&
            !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {
                bbr->mode = BBR_PROBE_RTT;  /* dip, drain queue */
        if(((struct inet_sock*)sk)->inet_dport == 3000)
        {
        printk("mode 2 mode=%d,min_rtt_us=%d,full_bw=%d,cycle_idx=%d,pacing_gain=%d,cwnd_gain=%d,rtt_cnt=%d\r\n",bbr->mode,bbr->min_rtt_us,bbr->full_bw,bbr->cycle_idx,bbr->pacing_gain,bbr->cwnd_gain,bbr->rtt_cnt);
        }

                bbr->pacing_gain = BBR_UNIT;
.....
}

static void bbr_main(struct sock *sk, const struct rate_sample *rs)
{
struct bbr *bbr = inet_csk_ca(sk);
u32 bw;

bbr_update_model(sk, rs);

bw = bbr_bw(sk);
bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);
bbr_set_tso_segs_goal(sk);
bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);
if(((struct inet_sock*)sk)->inet_dport == 3000)
{
printk("main mode=%d,min_rtt_us=%d,cur_bw=%d,cycle_idx=%d,pacing_gain=%d,cwnd_gain=%d,rtt_cnt=%d,snd_cwnd=%d, snd_nxt=%u,snd_una=%u\r\n",bbr->mode,bbr->min_rtt_us,bbr_bw(sk),bbr->cycle_idx,bbr->pacing_gain,bbr->cwnd_gain,bbr->rtt_cnt,tcp_sk(sk)->snd_cwnd, tcp_sk(sk)->snd_nxt, tcp_sk(sk)->snd_una );
}

}

除了四個狀態轉換的地方，因爲咱們收到ack，都會進入bbr_main,因此咱們也在這個地方加了打印。

打印結果以下：

[11241.360364] mode 2 mode=3,min_rtt_us=553,full_bw=0,cycle_idx=0,pacing_gain=0,cwnd_gain=0,rtt_cnt=0
[11241.360373] main mode=3,min_rtt_us=553,cur_bw=0,cycle_idx=0,pacing_gain=256,cwnd_gain=256,rtt_cnt=0,snd_cwnd=4
[11241.360377] mode 2 mode=0,min_rtt_us=553,full_bw=0,cycle_idx=0,pacing_gain=256,cwnd_gain=256,rtt_cnt=0
[11241.367057] main mode=0,min_rtt_us=553,cur_bw=91528,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=1,snd_cwnd=20
[11241.369053] main mode=0,min_rtt_us=553,cur_bw=109875,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=2,snd_cwnd=33
[11241.375055] main mode=0,min_rtt_us=553,cur_bw=267597,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=3,snd_cwnd=56
[11241.383053] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=4,snd_cwnd=79
[11241.392054] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=5,snd_cwnd=102
[11241.401053] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=6,snd_cwnd=125
[11241.409054] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=7,snd_cwnd=148
[11241.417055] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=8,snd_cwnd=171
[11241.426086] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=9,snd_cwnd=171
[11241.434054] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=10,snd_cwnd=171
[11241.442055] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=11,snd_cwnd=171
。。。。。


中間只增長rtt_cnt的部分省略：

[11243.420044] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=246,snd_cwnd=187
[11243.429045] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=247,snd_cwnd=187
[11243.437044] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=248,snd_cwnd=187
[11243.446046] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=249,snd_cwnd=187
[11243.454046] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=250,snd_cwnd=187 [11243.502075] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=251,snd_cwnd=195------------rtt從8ms變成48ms [11243.544064] main mode=0,min_rtt_us=553,cur_bw=364377,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=252,snd_cwnd=195-----------42ms
[11243.586056] main mode=0,min_rtt_us=553,cur_bw=39570,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=253,snd_cwnd=195------------42ms
[11243.628057] main mode=0,min_rtt_us=553,cur_bw=50749,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=254,snd_cwnd=195------------42ms
[11243.670058] main mode=0,min_rtt_us=553,cur_bw=50749,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=255,snd_cwnd=195------------42ms [11243.753064] main mode=0,min_rtt_us=553,cur_bw=50749,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=256,snd_cwnd=195------------83ms [11243.795063] mode 2 mode=1,min_rtt_us=553,full_bw=109875,cycle_idx=0,pacing_gain=739,cwnd_gain=739,rtt_cnt=257----------------從startup模式切換到drain模式 [11243.795068] mode 2 mode=2,min_rtt_us=553,full_bw=109875,cycle_idx=0,pacing_gain=88,cwnd_gain=739,rtt_cnt=257----------------馬上從drain模式切換到探測bw模式
[11243.795072] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=257,snd_cwnd=14
[11243.837061] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=258,snd_cwnd=14
[11243.879063] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=259,snd_cwnd=14
[11243.921061] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=260,snd_cwnd=14
[11243.963063] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=261,snd_cwnd=14
[11244.005063] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=262,snd_cwnd=10
[11244.047065] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=263,snd_cwnd=14
[11244.089065] main mode=2,min_rtt_us=553,cur_bw=50749,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=264,snd_cwnd=14
[11244.131066] main mode=2,min_rtt_us=553,cur_bw=50352,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=265,snd_cwnd=14
[11244.173065] main mode=2,min_rtt_us=553,cur_bw=50352,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=266,snd_cwnd=14
[11244.215069] main mode=2,min_rtt_us=553,cur_bw=50352,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=267,snd_cwnd=14
[11244.257067] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=268,snd_cwnd=8
[11244.299067] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=269,snd_cwnd=8
[11244.341077] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=270,snd_cwnd=8
[11244.383073] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=271,snd_cwnd=8
[11244.425070] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=272,snd_cwnd=8
[11244.467071] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=273,snd_cwnd=8
[11244.509073] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=274,snd_cwnd=8
[11244.551076] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=275,snd_cwnd=8
[11244.593078] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=276,snd_cwnd=8
[11244.635078] main mode=2,min_rtt_us=553,cur_bw=5594,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=277,snd_cwnd=8
[11244.677079] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=278,snd_cwnd=8
[11244.719083] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=279,snd_cwnd=8
[11244.761082] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=280,snd_cwnd=8

黃色顏色部分就是咱們忽然使用delay_ack的地方，這個能夠經過先後兩個ack的時間相差來肯定。

能夠看到，bbr反應至關劇烈，在一個RTO左右的時間，將snd_cwnd迅速下降，

ps：初始狀態並非0，這個並非咱們代碼打印有問題，而是內核中真的就這麼執行的，我的以爲是個bug，具體分析能夠參見《https://www.cnblogs.com/10087622blog/p/10412440.html》。

咱們復現的帶寬圖：

對應的rtt圖：

怎麼解釋在10s附近，帶寬增加不少呢？繼續根據咱們prink的數據來分析：

[11251.018193] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=429,snd_cwnd=8
[11251.060199] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=430,snd_cwnd=8
[11251.102199] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=431,snd_cwnd=8
[11251.144200] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=432,snd_cwnd=8
[11251.186198] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=433,snd_cwnd=8
[11251.228199] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=434,snd_cwnd=8
[11251.270202] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=435,snd_cwnd=8
[11251.312199] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=436,snd_cwnd=8
[11251.354205] main mode=2,min_rtt_us=553,cur_bw=3197,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=437,snd_cwnd=8
[11251.396210] mode 2 mode=3,min_rtt_us=41442,full_bw=109875,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=438------mode從探測bw到探測rtt
[11251.396215] main mode=3,min_rtt_us=41442,cur_bw=3197,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=438,snd_cwnd=4----取4個mss來發包
[11251.438203] main mode=3,min_rtt_us=41442,cur_bw=3197,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=439,snd_cwnd=4----穩定的min_rtt變成41442us
[11251.480205] main mode=3,min_rtt_us=41442,cur_bw=3197,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=440,snd_cwnd=4
[11251.522214] main mode=3,min_rtt_us=41442,cur_bw=3197,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=441,snd_cwnd=4
[11251.564202] main mode=3,min_rtt_us=41442,cur_bw=3197,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=442,snd_cwnd=4----探測持續200ms，符合代碼
[11251.606206] mode 2 mode=2,min_rtt_us=41442,full_bw=109875,cycle_idx=1,pacing_gain=256,cwnd_gain=256,rtt_cnt=443-------mode從探測rtt回到探測bw
[11251.606215] main mode=2,min_rtt_us=41442,cur_bw=3197,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=443,snd_cwnd=12---開始指數增加咱們的cwnd
[11251.648208] main mode=2,min_rtt_us=40906,cur_bw=4795,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=444,snd_cwnd=24
[11251.690214] main mode=2,min_rtt_us=39861,cur_bw=9589,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=445,snd_cwnd=48
[11251.732220] main mode=2,min_rtt_us=36713,cur_bw=19184,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=446,snd_cwnd=90
[11251.774230] main mode=2,min_rtt_us=31462,cur_bw=35975,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=447,snd_cwnd=142
[11251.816239] main mode=2,min_rtt_us=25016,cur_bw=50376,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=448,snd_cwnd=160
[11251.858244] main mode=2,min_rtt_us=25016,cur_bw=50376,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=449,snd_cwnd=160
[11251.900231] main mode=2,min_rtt_us=23843,cur_bw=50376,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=450,snd_cwnd=150
[11251.942238] main mode=2,min_rtt_us=23843,cur_bw=50376,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=451,snd_cwnd=154
[11251.984226] main mode=2,min_rtt_us=23841,cur_bw=50376,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=452,snd_cwnd=154
[11252.026236] main mode=2,min_rtt_us=23841,cur_bw=50376,cycle_idx=4,pacing_gain=256,cwnd_gain=512,rtt_cnt=453,snd_cwnd=154
[11252.068228] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=5,pacing_gain=256,cwnd_gain=512,rtt_cnt=454,snd_cwnd=154
[11252.110231] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=6,pacing_gain=256,cwnd_gain=512,rtt_cnt=455,snd_cwnd=154
[11252.152229] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=7,pacing_gain=256,cwnd_gain=512,rtt_cnt=456,snd_cwnd=154
[11252.194235] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=0,pacing_gain=320,cwnd_gain=512,rtt_cnt=457,snd_cwnd=154
[11252.236229] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=1,pacing_gain=192,cwnd_gain=512,rtt_cnt=458,snd_cwnd=150
[11252.278264] main mode=2,min_rtt_us=23828,cur_bw=50382,cycle_idx=2,pacing_gain=256,cwnd_gain=512,rtt_cnt=459,snd_cwnd=154
[11252.320232] main mode=2,min_rtt_us=23828,cur_bw=50407,cycle_idx=3,pacing_gain=256,cwnd_gain=512,rtt_cnt=460,snd_cwnd=154