softlockup檢測(watchdog)原理(用於檢測系統調度是否正常)

softlockup(watchdog)用於檢測系統調度是否正常，即軟鎖的狀況，當發生softlockup時，內核不能調度，但還能響應中斷，對用戶的表現可能爲：能ping通，但沒法登錄系統，沒法進行正常操做。
其基本原理爲：爲每一個CPU啓動一個內核線程(watchdog/x)，此線程爲優先級最高的實時線程，在該線程獲得調度時，會更新相應的計數(時間戳)，同時會啓動定時器，當定時器到期時檢查相應的時間戳，若是超過指定時間，都沒有更新，則說明這段時間內都沒有發生調度(由於此線程優先級最高)，則打印相應告警或根據配置能夠進入panic流程。
基本代碼分析(2.6.32)
rest_init->kernel_init->lockup_detector_init->cpu_callback->watchdog_prepare_cpu(初始化watchdog定時器):

點擊(此處)摺疊或打開

1. static int watchdog_prepare_cpu(int cpu)
   

2. {
   

3.     struct hrtimer *hrtimer = &per_cpu(watchdog_hrtimer, cpu);
   

4. 
   

5.     WARN_ON(per_cpu(softlockup_watchdog, cpu));
   

6.     hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);//初始化高精度定時器
   

7.     hrtimer->function = watchdog_timer_fn;//設置定時器處理函數
   

8. 
   

9.     return 0;
   

10. }

看門狗定時器處理函數：

點擊(此處)摺疊或打開

1. static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
   

2. {

3. //獲取計數watchdog_touch_ts，該計數在watchdog內核線程被調度時更新
   

4.     unsigned long touch_ts = __get_cpu_var(watchdog_touch_ts);
   

5.     struct pt_regs *regs = get_irq_regs();
   

6.     int duration;
   

7. 
   

8.     /* kick the hardlockup detector */

9. //增長中斷計數，證實沒有發生硬鎖(關中斷死鎖)
   

10.     watchdog_interrupt_count();
    

11. 
    

12.     /* kick the softlockup detector */

13. //喚醒wathdog內核線程
    

14.     wake_up_process(__get_cpu_var(softlockup_watchdog));
    

15. 
    

16.     /* .. and repeat */

17. //重啓定時器
    

18.     hrtimer_forward_now(hrtimer, ns_to_ktime(get_sample_period()));
    

19.     if (touch_ts == 0) {
    

20.         if (unlikely(__get_cpu_var(softlockup_touch_sync))) {
    

21.             /*
    

22.              * If the time stamp was touched atomically
    

23.              * make sure the scheduler tick is up to date.
    

24.              */
    

25.             __get_cpu_var(softlockup_touch_sync) = false;
    

26.             sched_clock_tick();
    

27.         }
    

28.         __touch_watchdog();
    

29.         return HRTIMER_RESTART;
    

30.     }
    

31. 
    

32.     /* check for a softlockup
    

33.      * This is done by making sure a high priority task is
    

34.      * being scheduled. The task touches the watchdog to
    

35.      * indicate it is getting cpu time. If it hasn't then
    

36.      * this is a good indication some task is hogging the cpu
    

37.      */

38. //判斷是否發生了軟鎖，原理是判斷touch_ts(時間戳)是否超過必定時間沒有更新
    

39.     duration = is_softlockup(touch_ts);
    

40.     if (unlikely(duration)) {
    

41.         /* only warn once */
    

42.         if (__get_cpu_var(soft_watchdog_warn) == true)
    

43.             return HRTIMER_RESTART;
    

44. //發生了軟鎖後，進行一些列的信息記錄和告警。
    

45.         printk(KERN_EMERG "BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
    

46.             smp_processor_id(), duration,
    

47.             current->comm, task_pid_nr(current));
    

48.         print_modules();
    

49.         print_irqtrace_events(current);
    

50.         if (regs)
    

51.             show_regs(regs);
    

52.         else
    

53.             dump_stack();
    

54. //若是配置了softlockup_panic(proc中配置)，則panic
    

55.         if (softlockup_panic)
    

56.             panic("softlockup: hung tasks");
    

57.         __get_cpu_var(soft_watchdog_warn) = true;
    

58.     } else
    

59.         __get_cpu_var(soft_watchdog_warn) = false;
    

60. 
    

61.     return HRTIMER_RESTART;
    

62. }

啓動看門狗，即建立watchdog內核線程。

點擊(此處)摺疊或打開

1. static int watchdog_enable(int cpu)
   

2. {
   

3.     struct task_struct *p = per_cpu(softlockup_watchdog, cpu);
   

4.     int err = 0;
   

5. 
   

6.     /* enable the perf event */
   

7.     err = watchdog_nmi_enable(cpu);
   

8. 
   

9.     /* Regardless of err above, fall through and start softlockup */
   

10. 
    

11.     /* create the watchdog thread */
    

12.     if (!p) {

13. //建立watchdog內核線程
    

14.         p = kthread_create(watchdog, (void *)(unsigned long)cpu, "watchdog/%d", cpu);
    

15.         if (IS_ERR(p)) {
    

16.             printk(KERN_ERR "softlockup watchdog for %i failed\n", cpu);
    

17.             if (!err)
    

18.                 /* if hardlockup hasn't already set this */
    

19.                 err = PTR_ERR(p);
    

20.             goto out;
    

21.         }
    

22.         kthread_bind(p, cpu);
    

23.         per_cpu(watchdog_touch_ts, cpu) = 0;
    

24.         per_cpu(softlockup_watchdog, cpu) = p;
    

25.         wake_up_process(p);
    

26.     }
    

27. 
    

28. out:
    

29.     return err;
    

30. }

watchdog內核線程執行主函數，主要是要更新計數(時間戳)

點擊(此處)摺疊或打開

1. static int watchdog(void *unused)
   

2. {

3. //設置爲最高優先級
   

4.     struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
   

5.     struct hrtimer *hrtimer = &__raw_get_cpu_var(watchdog_hrtimer);
   

6. //設置爲實時線程
   

7.     sched_setscheduler(current, SCHED_FIFO, &param);
   

8. 
   

9.     /* initialize timestamp */

10. //初始化計數(時間戳)
    

11.     __touch_watchdog();
    

12. 
    

13.     /* kick off the timer for the hardlockup detector */
    

14.     /* done here because hrtimer_start can only pin to smp_processor_id() */

15. //啓動定時器，用於檢測是否發生軟鎖
    

16.     hrtimer_start(hrtimer, ns_to_ktime(get_sample_period()),
    

17.          HRTIMER_MODE_REL_PINNED);
    

18. //睡眠
    

19.     set_current_state(TASK_INTERRUPTIBLE);
    

20.     /*
    

21.      * Run briefly once per second to reset the softlockup timestamp.
    

22.      * If this gets delayed for more than 60 seconds then the
    

23.      * debug-printout triggers in watchdog_timer_fn().
    

24.      */
    

25.     while (!kthread_should_stop()) {

26. //更新計數
    

27.         __touch_watchdog();
    

28.         schedule();
    

29. 
    

30.         if (kthread_should_stop())
    

31.             break;
    

32. 
    

33.         set_current_state(TASK_INTERRUPTIBLE);
    

34.     }
    

35.     __set_current_state(TASK_RUNNING);
    

36. 
    

37.     return 0;
    

38. }

判斷是否發生軟鎖：is_softlockup

點擊(此處)摺疊或打開

1. static int is_softlockup(unsigned long touch_ts)
   

2. {
   

3.     unsigned long now = get_timestamp(smp_processor_id());
   

4. 
   

5.     /* Warn about unreasonable delays: */

6. //檢測計數多久沒有更新了，若是超過了60s，則表示發生了軟鎖
   

7.     if (time_after(now, touch_ts + softlockup_thresh))
   

8.         return now - touch_ts;
   

9. 
   

10.     return 0;
    

11. }