linux 下強大的 JSON 解析命令 jq

介紹

jq is like sed for JSON data - you can use it to slice and filter and map and transform structured data with the same ease that sed, awk, grep and friends let you play with text.

jq 能夠對 JSON 數據進行切片、過濾、映射和轉換，和sed, awk, grep 命令同樣簡單好用。

jq is written in portable C, and it has zero runtime dependencies. You can download a single binary, scp it to a far away machine of the same type, and expect it to work.

jq 是用 C語言編寫的，沒有運行時依賴。獨立二進制文件，能夠使用 scp 複製到其餘服務器直接運行。

安裝

Linux

# 下載 https://stedolan.github.io/jq/download/
$ wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /usr/local/bin/jq

# 執行權限
$ chmod +x /usr/local/bin/jq

OS X

$ brew install jq

使用

以 njmon 的 json 輸出來演示 jq 的經常使用方式，高級用法見官方手冊。

jq "."

以 json格式化輸出。

$ njmon -s 1 -c 1 | jq '.'
{
  "timestamp": {
    "datetime": "2020-03-14T19:59:22",
    "UTC": "2020-03-14T11:59:22",
    "snapshot_seconds": 1,
    "snapshot_maxloops": 1,
    "snapshot_loop": 0
  },
 ..... 省略其餘輸出
}

指定 key 查詢

$ njmon -s 1 -c 1 | jq '.timestamp'
{
  "datetime": "2020-03-14T20:14:00",
  "UTC": "2020-03-14T12:14:00",
  "snapshot_seconds": 1,
  "snapshot_maxloops": 1,
  "snapshot_loop": 0
}

$ njmon -s 1 -c 1 | jq '.timestamp.datetime'
"2020-03-14T20:14:43"

--raw-output/-r

標準輸出，即不格式化爲帶引號的json 字符串

$ njmon -s 1 -c 1 | jq '.timestamp.datetime'
"2020-03-14T20:14:43"

$ njmon -s 1 -c 1 | jq -r '.timestamp.datetime'
2020-03-14T20:14:43

@csv：格式化輸出

還有其餘格式方式： @html,@sh,@base64,@base64d等。

$ njmon -s 1 -c 1 | jq -r 'keys' | jq -r '@csv'
"cpu_total","cpuinfo","cpus","disks","filesystems","identity","lscpu","networks","os_release","proc_meminfo","proc_version","proc_vmstat","stat_counters","timestamp","uptime"

數組

$ njmon -s 1 -c 1 | jq '.cpus'
{
  "cpu0": {
    "user": 0.999,
    "nice": 0,
    "sys": 0.999,
    "idle": 99.87,
    "iowait": 0,
    "hardirq": 0,
    "softirq": 0,
    "steal": 0,
    "guest": 0,
    "guestnice": 0
  },
  "cpu1": {
    "user": 2.996,
    "nice": 0,
    "sys": 0.999,
    "idle": 97.873,
    "iowait": 0,
    "hardirq": 0,
    "softirq": 0,
    "steal": 0,
    "guest": 0,
    "guestnice": 0
  }
}

$ njmon -s 1 -c 1 | jq '.cpus[]'
{
  "user": 0.999,
  "nice": 0,
  "sys": 0.999,
  "idle": 99.92,
  "iowait": 0,
  "hardirq": 0,
  "softirq": 0,
  "steal": 0,
  "guest": 0,
  "guestnice": 0
}
{
  "user": 2.998,
  "nice": 0,
  "sys": 0.999,
  "idle": 97.922,
  "iowait": 0,
  "hardirq": 0,
  "softirq": 0,
  "steal": 0,
  "guest": 0,
  "guestnice": 0
}

函數

keys：獲取JSON中的key信息

$ njmon -s 1 -c 1 | jq 'keys'
[
  "cpu_total",
  "cpuinfo",
  "cpus",
  "disks",
  "filesystems",
  "identity",
  "lscpu",
  "networks",
  "os_release",
  "proc_meminfo",
  "proc_version",
  "proc_vmstat",
  "stat_counters",
  "timestamp",
  "uptime"
]

select(boolean_expression)：搜索指定條件內容

$ njmon -s 1 -c 1 | jq '.filesystems[] | select(.fs_dir == "/home")'
{
  "fs_dir": "/home",
  "fs_type": "ext4",
  "fs_opts": "rw,seclabel,relatime,data=ordered",
  "fs_freqs": 0,
  "fs_passno": 0,
  "fs_bsize": 4096,
  "fs_size_mb": 11502,
  "fs_free_mb": 11448,
  "fs_used_mb": 54,
  "fs_full_percent": 0.463,
  "fs_avail": 10842,
  "fs_files": 757392,
  "fs_files_free": 757110,
  "fs_namelength": 255
}

has(key)：判斷是否存在某個key

$ njmon -s 1 -c 1 | jq 'has("cpus")'
true

$ njmon -s 1 -c 1 | jq 'has("cpus2")'
false

length：查看輸出長度或個數

# 5個字段
$ njmon -s 1 -c 1 | jq '.timestamp | length'
5

# 長度19
$ njmon -s 1 -c 1 | jq '.timestamp.datetime | length'
19

實際場景

nacos 工具腳本

在作自動部署腳本時，須要寫個 Nacos 工具腳本，實現操做 Nacos 實例狀態查看、下線、上線等。針對 Nacos 的 json 結構數據，使用 jq 來進行處理。腳本以下：

nacosUtils.sh

#!/bin/bash
# Nacos工具腳本

# 使用方法
usage() {
   echo "Usage: sh $0 {instance|instances|offline|online} URL NAMESPACE_ID SERVICE_NAME IP PORT [FORCE]"
   echo "\n"
   echo "\t {instance|instances|offline|online} 方法名稱"
   echo "\t\t -instance：查詢某個健康實例狀態，true表示在線狀態，false表示下線狀態, 返回空表示實例不存在"
   echo "\t\t -instances：查詢健康實例總數"
   echo "\t SERVICE_NAME 系統名稱"
   echo "\t IP 實例IP"
   echo "\t PORT 實例啓動端口號"
   echo "\t FORCE 方法爲offline時，true表示強制下線，非true表示校驗是否多於2個健康實例"
   echo "\n"
   echo "示例："
   echo "\t 啓動：sh $0 instance http://127.0.0.1/nacos ****** serviceName x.x.x.x 8090"
}

# 判斷參數
if [ $# -lt 4 ];
then
  usage
  exit 1
fi

# nacos url
URL=$2
# nacos 命名空間
NAMESPACE_ID=$3
# 系統名稱
SERVICE_NAME=$4
# 實例 IP地址
IP=$5
# 端口
PORT=$6
# 強制下線標識
FORCE=$7

# 查看實例url
instanceListUrl="$URL/v1/ns/catalog/instances?serviceName=$SERVICE_NAME&clusterName=DEFAULT&pageSize=10&pageNo=1&namespaceId=$NAMESPACE_ID"
# 更新實例
instanceUrl="$URL/v1/ns/instance?serviceName=$SERVICE_NAME&ip=$IP&port=$PORT&namespaceId=$NAMESPACE_ID"

# 查詢健康且上線的實例個數
function instances() {
  echo `curl -X GET -s $instanceListUrl | jq -r ".list[] | select(.healthy == true) | select(.enabled == true) | .ip" | wc -l`
}

# 查詢健康實例狀態  在線：true；下線：false, 沒有查詢到: 空
function instance() {
  echo `curl -X GET -s $instanceListUrl | jq -r ".list[] | select(.ip == \"$IP\") | select(.port == $PORT) | select(.healthy == true) | .enabled"`
}

# 下線  健康實例總數爲1時，不可下線惟一的服務
function offline() {
  if [ "X$FORCE" != "Xtrue" ];then
    counts=`instances`
    if [ $counts -lt 2 ];then
      curl -X GET -s $instanceListUrl | jq .
      echo "$SERVICE_NAME 實例總數小於2個，不可下線惟一的服務！！！"
      exit 1
    fi
  fi

  result=`curl -X PUT -s "$instanceUrl&enabled=false"`
  echo $result
}

# 上線
function online() {
  enabledFlag=`instance`
  if [ "X$enabledFlag" == "Xtrue" ]; then
    curl -X GET -s $instanceListUrl | jq .
    echo "$SERVICE_NAME 實例爲上線狀態，無需執行上線！！！"
  else
    result=`curl -X PUT -s "$instanceUrl&enabled=true"`
  fi
  echo $result
}

case "$1" in
        'instance')
            instance
            ;;

        'instances')
            instances
            ;;

        'offline')
            offline
            ;;

        'online')
            online
            ;;

        *)
            usage
            exit 1
            ;;
esac

exit 0

njmon 結果格式化

njmon與nmon相似，但輸出爲 JSON 格式，用於服務器性能統計。

$ njmon -s 1 -c 1 | jq .
{
  "timestamp": {
    "datetime": "2020-03-14T19:59:22",
    "UTC": "2020-03-14T11:59:22",
    "snapshot_seconds": 1,
    "snapshot_maxloops": 1,
    "snapshot_loop": 0
  },
 ..... 省略其餘輸出
}

擴展閱讀

• jq 官網：https://stedolan.github.io/jq
• jq 在線運行：https://jqplay.org/