crash commands to analyze kdumps

1. list processes grouped by process state

crash> ps -S
  RU: 38
  IN: 1127
  UN: 23

2.  search on the stack of all processes

crash>  for UN bt | grep '#1 ' | awk '{print $3}' | sort | uniq -c | sort -r -n
      9 io_schedule
      7 schedule_timeout
      5 __mutex_lock_slowpath
      1 xlog_state_get_iclog_space
      1 xfs_ioend_wait

3. memory info

crash> kmem -i
                 PAGES        TOTAL      PERCENTAGE
    TOTAL MEM  32934221     125.6 GB         ----
         FREE  29938211     114.2 GB   90% of TOTAL MEM

4. disk info

crash> dev -d
MAJOR GENDISK            NAME       REQUEST_QUEUE      TOTAL ASYNC  SYNC   DRV
    8 ffff880fdae57800   sda        ffff880fdc2ad538       0     0     0     0
    8 ffff880fdae58000   sdc        ffff880fde7b4e90       0     0     0     0
    8 ffff880fdb5f3400   sde        ffff880fdc2ace50       0     0     0     0

5. look into mounted fs

crash> mount ffff880fd8bd7800
    VFSMOUNT         SUPERBLK     TYPE   DEVNAME   DIRNAME
ffff880fdfcc76c0 ffff880fd8bd7800 xfs    /dev/mapper/xxx
crash> struct -x super_block.s_frozen ffff880fd8bd7800
  s_frozen = 0x0

6. display BT

crash> bt ffff880fbce6e140
PID: 14003  TASK: ffff880fbce6e140  CPU: 5   COMMAND: "java"
 #0 [ffff880fcf361ae0] schedule at ffffffff814652b9
 #1 [ffff880fcf361c28] __mutex_lock_slowpath at ffffffff81466488
 #2 [ffff880fcf361c88] mutex_lock at ffffffff81465f1a
 #3 [ffff880fcf361ca0] do_lookup at ffffffff8116a1b8
 #4 [ffff880fcf361d10] path_lookupat at ffffffff8116d98c
 #5 [ffff880fcf361da0] do_path_lookup at ffffffff8116df1f
 #6 [ffff880fcf361dd0] user_path_at_empty at ffffffff8116eecf
 #7 [ffff880fcf361ea0] vfs_fstatat at ffffffff81163570
 #8 [ffff880fcf361ef0] sys32_stat64 at ffffffff8103f48f
 #9 [ffff880fcf361f80] sysenter_dispatch at ffffffff81470ab0

7. struct commands

crash> struct -x dentry.d_inode ffff881394074800
  d_inode = 0xffff881fcb88bd90

crash> files -d ffff881394074800
     DENTRY           INODE           SUPERBLK     TYPE PATH
ffff881394074800 ffff881fcb88bd90 ffff880fd8bd7800 DIR  /mnt/xxx

crash> struct -x inode.i_mutex 0xffff881fcb88bd90
  i_mutex = {
    count = {
      counter = 0xfffffffd
    },