ElasticSearch中"distinct","count"和"group by"的實現

最近在業務中須要使用ES來進行數據查詢，在某些場景下須要對數據進行去重，以及去重後的統計。爲了方便你們理解，特地從SQL角度，方便你們可以理解ES查詢語句。

 

1 - distinct

SELECT DISTINCT(user_id) FROM table WHERE user_id_type = 3;

 

{
  "query": {
    "term": {
      "user_id_type": 3
    }
  },
  "collapse": {
    "field": "user_id"
  }
}

　　

{
  ...
  "hits": {
    "hits": [
      {
        "_index": "es_qd_mkt_visitor_packet_dev_v1_20180621",
        "_type": "ad_crowd",
        "_source": {
          "user_id": "wx2af8414b502d4ca2_oHtrD0Vxv-_8c678figJNHmtaVQQ",
          "user_id_type": 3
        },
        "fields": {
          "user_id": [
            "wx2af8414b502d4ca2_oHtrD0Vxv-_8c678figJNHmtaVQQ"
          ]
        }
      }
    ]
  }
}

　　

總結：使用collapse字段後，查詢結果中[hits]中會出現[fields]字段，其中包含了去重後的user_id

2 - count + distinct

SELECT COUNT(DISTINCT(user_id)) FROM table WHERE user_id_type = 3;

　　

{
  "query": {
    "term": {
      "user_id_type": 3
    }
  },
  "aggs": {
    "count": {
      "cardinality": {
        "field": "user_id"
      }
    }
  }
}

　

{
  ...
  "hits": {
  ...
  },
  "aggregations": {
    "count": {
      "value": 121
    }
  }
}

　　

總結：aggs中cardinality的字段表明須要distinct的字段

3 - count + group by

SELECT COUNT(user_id) FROM table GROUP BY user_id_type;

　　

{
  "aggs": {
    "user_type": {
      "terms": {
        "field": "user_id_type"
      }
    }
  }
}

　　

{
  ...
  "hits": {
    ...
  },
  "aggregations": {
    "user_type": {
      ...
      "buckets": [
        {
          "key": 4,
          "doc_count": 1220
        },
        {
          "key": 3,
          "doc_count": 488
        }
      ]
    }
  }
}

　　

總結：aggs中terms的字段表明須要gruop by的字段

4 - count + distinct + group by

SELECT COUNT(DISTINCT(user_id)) FROM table GROUP BY user_id_type;

　　

{
  "aggs": {
    "user_type": {
      "terms": {
        "field": "user_id_type"
      },
      "aggs": {
        "count": {
          "cardinality": {
            "field": "user_id"
          }
        }
      }
    }
  }
}
{
  ...
  "hits": {
    ...
  },
  "aggregations": {
    "user_type": {
      ...
      "buckets": [
        {
          "key": 4,
          "doc_count": 1220, //去重前數據1220條
          "count": {
            "value": 276 //去重後數據276條
          }
        },
        {
          "key": 3,
          "doc_count": 488, //去重前數據488條
          "count": {
            "value": 121 //去重後數據121條
          }
        }
      ]
    }
  }
}

　　

4 - count + distinct + group by

SELECT COUNT(DISTINCT(user_id)) FROM table WHERE user_id_type = 2 GROUP BY user_id;

　　

總結：對於既有group by又有distinct的查詢要求，須要在aggs中嵌套子aggs

5 - 注意事項

collapse關鍵字

1. 摺疊功能ES5.3版本以後才發佈的。
2. 聚合&摺疊只能針對keyword類型有效