容器運行時docker和containerd兼容問題

背景

k8s 1.20 默認採用containerd做爲容器運行時

• https://kubernetes.io/zh/blog...

containerd 和 docker兼容問題

• 若是項目中須要用到底層 docker-api 則須要考慮兼容問題

k8s-mon項目 簡介

• 是什麼：滴滴夜鶯Kubernetes monitor
• 項目地址 https://github.com/n9e/k8s-mon

k8s-mon爲何要兼容containerd

由於在k8s-mon中採集容器基礎資源指標須要和夜鶯的服務樹打通
須要獲取容器服務樹標籤N9E_NID

默認版本是採用docker inspect接口獲取的

package collect

import (
    "context"
    "github.com/docker/docker/api/types"
    "github.com/docker/docker/client"
)

func getLabelMapByDockerSdk(nidLabelName string) (map[string]map[string]string, error) {
    ctx := context.Background()
    cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
    if err != nil {
        return nil, err
    }
    defer cli.Close()
    containers, err := cli.ContainerList(ctx, types.ContainerListOptions{})
    if err != nil {
        return nil, err
    }

    insM := make(map[string]map[string]string)
    whiteKeyM := map[string]struct{}{
        nidLabelName: {},
    }
    for _, container := range containers {
        podName := container.Labels["io.kubernetes.pod.name"]

        if podName == "" {
            continue
        }

        lastM, loaded := insM[podName]
        if !loaded {
            lastM = make(map[string]string)
        }

        for k, v := range container.Labels {

            if _, found := whiteKeyM[k]; !found {
                continue
            }
            lastM[k] = v
        }

        insM[podName] = lastM

    }
    return insM, nil
}

須要兼容下containerd的接口獲取

package collect

import (
    "context"
    "errors"
    "github.com/containerd/containerd"
)

func getLabelMapByContainerdSdk(nidLabelName string) (map[string]map[string]string, error) {
    client, err := containerd.New("/run/containerd/containerd.sock", containerd.WithDefaultNamespace("k8s.io"))

    if err != nil {
        return nil, err
    }

    defer client.Close()

    context := context.Background()
    cers, err := client.Containers(context)
    if err != nil {
        return nil, err
    }
    if len(cers) == 0 {
        return nil, errors.New("got zero containers on this node")
    }

    insM := make(map[string]map[string]string)
    whiteKeyM := map[string]struct{}{
        nidLabelName: {},
    }

    for _, c := range cers {
        labels, err := c.Labels(context)
        if err != nil {
            continue
        }

        podName := labels["io.kubernetes.pod.name"]
        if podName == "" {
            continue
        }

        lastM, loaded := insM[podName]
        if !loaded {
            lastM = make(map[string]string)
        }

        for k, v := range labels {

            if _, found := whiteKeyM[k]; !found {
                continue
            }
            lastM[k] = v
        }

        insM[podName] = lastM

    }
    return insM, nil

}

k8s-mon獲取容器tag時須要適配，默認採用docker-api，失敗再嘗試containerd-api

docker 鏡像轉成containerd的

ctr 命令執行時須要帶上 --namespace k8s.io

docker build

docker build -t k8s-mon:v1 . && docker save  k8s-mon > a.tar

ctr 導入docker 鏡像

ctr --namespace k8s.io images import a.tar

ctr 給鏡像打tag = docker tag 4d8690b7763d registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.1.0

ctr --namespace k8s.io images tag docker.io/library/k8s-mon:v1 k8s-mon:v1

ctr get鏡像 = docker images

[root@k8s-node01 ~]# ctr --namespace k8s.io images ls  
REF                                                                                                                                           TYPE                                                      DIGEST                                                                  SIZE      PLATFORMS                                                      LABELS                          
docker.io/eipwork/kuboard:latest                                                                                                              application/vnd.docker.distribution.manifest.v2+json      sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB  linux/amd64                                                    io.cri-containerd.image=managed 
docker.io/eipwork/kuboard@sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad                                             application/vnd.docker.distribution.manifest.v2+json      sha256:7e3d463e856d4d46f88b3241946554b144e3c285953e5055f6a6d6fa2f6360ad 69.0 MiB  linux/amd64                                                    io.cri-containerd.image=managed 
docker.io/eipwork/metrics-server:v0.3.7                                                                                                       application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB  linux/amd64,linux/arm64                                        io.cri-containerd.image=managed 
docker.io/eipwork/metrics-server@sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240                                      application/vnd.docker.distribution.manifest.list.v2+json sha256:2a8e43d89e434b6235858444bb0701c44bd03475d2937fdf6831a7bb4ee14240 20.1 MiB  linux/amd64,linux/arm64                                        io.cri-containerd.image=managed 
docker.io/library/k8s-mon:v1

ctr 獲取容器 = docker ps

[root@k8s-node01 ~]# ctr --namespace k8s.io c ls  
CONTAINER                                                           IMAGE                                                                                                                              RUNTIME                  
1d79feb8854e66f0c7c21739f7ee048edc1e278ce76d66c52981ea878d6eb231    registry.aliyuncs.com/k8sxio/pause@sha256:927d98197ec1141a368550822d18fa1c60bdae27b78b0c004f705f548c07814f                         io.containerd.runc.v2    
22baad7dc66daf480b8498fe8c31e546bc87f27b3b45807f6c8f942fccc9b686    sha256:919a16510f41bb12c86508529f915b123a5ff589aebb0a4355614a829aa47d62                                                            io.containerd.runc.v2    
2d0eefeec5fa5bf6b4bdc41933eb251b9550310b71f1cc300a2d05710cbcd5e7    quay.io/coreos/kube-state-metrics:v1.9.7                                                                                           io.containerd.runc.v2    
492a71bfeba379a6310095c888c21da99f1badb8253a9d57bb4536d4a2e31b82    swr.cn-east-2.myhuaweicloud.com/kuboard-dependency/node@sha256:304dd23bcda5216026f1601cb61395792249f1c58c98771198f6e517b0f5c96b    io.containerd.runc.v2    
4ebc6262d5217ea9b4f639051cb978398d2bcf38cd68b7bb83a20f270057bc2b    sha25

本地build 傳輸到遠端機器

docker build -t k8s-mon:v1 . && docker save  k8s-mon > a.tar && nc 172.20.70.215 444  < a.tar
# 遠端機器 
nc -l 444 >   a.tar && docker load < a.tar

push到阿里雲

docker push registry.cn-beijing.aliyuncs.com/n9e/k8s-mon:v2.0.7

登陸到阿里雲

docker login --username=燕小乙1875 registry.cn-beijing.aliyuncs.com

刪除殭屍容器

docker rmi $(docker images | grep "none" | awk '{print $3}')