Kubernetes 垃圾回收机制分析

为了回收系统上的资源kubelet有ImageGC和ContainerGC等功能对image和container进行回收；

根据kubelet代码对ContainerGC部分进行分析

相关的参数主要有:

• minimum-container-ttl-duration

• maximum-dead-containers-per-container

• minimum-container-ttl-duration

相对应的代码是:

1. //pkg/kubelet/container_gc.go

2. type ContainerGCPolicy struct {

3.    // 已经死掉的容器在机器上存留的时间

4.    MinAge time.Duration

5.    // 每个pod可以保留的死掉的容器

6.    MaxPerPodContainer int

7.    // 机器上最大的可以保留的死亡容器数量

8.    MaxContainers int

9. }

10. ...

11. func NewContainerGC(runtime Runtime, policy ContainerGCPolicy, sourcesReadyProvider SourcesReadyProvider) (ContainerGC, error) {

12.    if policy.MinAge < 0 {

13.        return nil, fmt.Errorf("invalid minimum garbage collection age: %v", policy.MinAge)

14.    }

15.    return &realContainerGC{

16.        runtime:              runtime,

17.        policy:               policy,

18.        sourcesReadyProvider: sourcesReadyProvider,

19.    }, nil

20. }

21. func (cgc *realContainerGC) GarbageCollect() error {

22.    return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), false)

23. }

24. func (cgc *realContainerGC) DeleteAllUnusedContainers() error {

25.    return cgc.runtime.GarbageCollect(cgc.policy, cgc.sourcesReadyProvider.AllReady(), true)

26. }

GarbageCollect方法里面的调用就是ContainerGC真正的逻辑所在, GarbageCollect函数是在pkg/kubelet/kubelet.go里面调用的，每隔一分钟会执行一次。GarbageCollect里面所调用的runtime的GarbageCollect函数是在pkg/kubelet/kuberuntime/kuberuntime_gc.go里面。

1. //pkg/kubelet/kuberuntime/kuberuntime_gc.go

2. func (cgc *containerGC) GarbageCollect(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {

3.    // Remove evictable containers

4.    if err := cgc.evictContainers(gcPolicy, allSourcesReady, evictNonDeletedPods); err != nil {

5.        return err

6.    }

7.    // Remove sandboxes with zero containers

8.    if err := cgc.evictSandboxes(evictNonDeletedPods); err != nil {

9.        return err

10.    }

11.    // Remove pod sandbox log directory

12.    return cgc.evictPodLogsDirectories(allSourcesReady)

13. }

第一步是驱逐容器

1. func (cgc *containerGC) evictContainers(gcPolicy kubecontainer.ContainerGCPolicy, allSourcesReady bool, evictNonDeletedPods bool) error {

2.    // Separate containers by evict units.

3.    evictUnits, err := cgc.evictableContainers(gcPolicy.MinAge)

4.    if err != nil {

5.        return err

6.    }

7.    // Remove deleted pod containers if all sources are ready.

8.    if allSourcesReady {

9.        for key, unit := range evictUnits {

10.            if cgc.isPodDeleted(key.uid) || evictNonDeletedPods {

11.                cgc.removeOldestN(unit, len(unit)) // Remove all.

12.                delete(evictUnits, key)

13.            }

14.        }

15.    }

16.    // Enforce max containers per evict unit.

17.    if gcPolicy.MaxPerPodContainer >= 0 {

18.        cgc.enforceMaxContainersPerEvictUnit(evictUnits, gcPolicy.MaxPerPodContainer)

19.    }

20.    // Enforce max total number of containers.

21.    if gcPolicy.MaxContainers >= 0 && evictUnits.NumContainers() > gcPolicy.MaxContainers {

22.        // Leave an equal number of containers per evict unit (min: 1).

23.        numContainersPerEvictUnit := gcPolicy.MaxContainers / evictUnits.NumEvictUnits()

24.        if numContainersPerEvictUnit < 1 {

25.            numContainersPerEvictUnit = 1

26.        }

27.        cgc.enforceMaxContainersPerEvictUnit(evictUnits, numContainersPerEvictUnit)

28.        // If we still need to evict, evict oldest first.

29.        numContainers := evictUnits.NumContainers()

30.        if numContainers > gcPolicy.MaxContainers {

31.            flattened := make([]containerGCInfo, 0, numContainers)

32.            for key := range evictUnits {

33.                flattened = append(flattened, evictUnits[key]...)

34.            }

35.            sort.Sort(byCreated(flattened))

36.            cgc.removeOldestN(flattened, numContainers-gcPolicy.MaxContainers)

37.        }

38.    }

39.    return nil

40. }

1.首先获取已经死掉的并且创建时间大于minage的容器

2.如果pod已经delete那么把属于这个pod的容器全部删除

3.如果设置了MaxPerPodContainer那么把MaxPerPodContainer之外数量的容器删除，这个值默认是1

4.如果设置MaxContainers那么再次对容器进行清理，这个值默认是-1也就是不清理的。首先会拿所有容器的数量除以pod的数量，这样会得到一个平均值，然后按照这个值对3进行再次处理。这个时候如果机器上的死亡容器的数量还大于MaxContainer那么直接按照时间对容器进行排序然后删除大于MaxContainer数量之外的容器。

下一步是清理机器上的sandbox

1. func (cgc *containerGC) evictSandboxes(evictNonDeletedPods bool) error {

2.    containers, err := cgc.manager.getKubeletContainers(true)

3.    if err != nil {

4.        return err

5.    }

6.    sandboxes, err := cgc.manager.getKubeletSandboxes(true)

7.    if err != nil {

8.        return err

9.    }

10.    sandboxesByPod := make(sandboxesByPodUID)

11.    for _, sandbox := range sandboxes {

12.        podUID := types.UID(sandbox.Metadata.Uid)

13.        sandboxInfo := sandboxGCInfo{

14.            id:         sandbox.Id,

15.            createTime: time.Unix(0, sandbox.CreatedAt),

16.        }

17.        // Set ready sandboxes to be active.

18.        if sandbox.State == runtimeapi.PodSandboxState_SANDBOX_READY {

19.            sandboxInfo.active = true

20.        }

21.        // Set sandboxes that still have containers to be active.

22.        hasContainers := false

23.        sandboxID := sandbox.Id

24.        for _, container := range containers {

25.            if container.PodSandboxId == sandboxID {

26.                hasContainers = true

27.                break

28.            }

29.        }

30.        if hasContainers {

31.            sandboxInfo.active = true

32.        }

33.        sandboxesByPod[podUID] = append(sandboxesByPod[podUID], sandboxInfo)

34.    }

35.    // Sort the sandboxes by age.

36.    for uid := range sandboxesByPod {

37.        sort.Sort(sandboxByCreated(sandboxesByPod[uid]))

38.    }

39.    for podUID, sandboxes := range sandboxesByPod {

40.        if cgc.isPodDeleted(podUID) || evictNonDeletedPods {

41.            // Remove all evictable sandboxes if the pod has been removed.

42.            // Note that the latest dead sandbox is also removed if there is

43.            // already an active one.

44.            cgc.removeOldestNSandboxes(sandboxes, len(sandboxes))

45.        } else {

46.            // Keep latest one if the pod still exists.

47.            cgc.removeOldestNSandboxes(sandboxes, len(sandboxes)-1)

48.        }

49.    }

50.    return nil

51. }

先获取机器上所有的容器和sandbox,如果pod的状态是0则致为active状态，如果此sandbox还有运行的container也认为是active状态，接着对sandbox进行排序，如果sandbox所属的pod的已经被删除那么删除所有的sandbox，如果pod还存在那么就留下最新的一个sandbox其他的都删除.

最后一步是清除container和pod日志

1. // evictPodLogsDirectories evicts all evictable pod logs directories. Pod logs directories

2. // are evictable if there are no corresponding pods.

3. func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {

4.    osInterface := cgc.manager.osInterface

5.    if allSourcesReady {

6.        // Only remove pod logs directories when all sources are ready.

7.        dirs, err := osInterface.ReadDir(podLogsRootDirectory)

8.        if err != nil {

9.            return fmt.Errorf("failed to read podLogsRootDirectory %q: %v", podLogsRootDirectory, err)

10.        }

11.        for _, dir := range dirs {

12.            name := dir.Name()

13.            podUID := types.UID(name)

14.            if !cgc.isPodDeleted(podUID) {

15.                continue

16.            }

17.            err := osInterface.RemoveAll(filepath.Join(podLogsRootDirectory, name))

18.            if err != nil {

19.                glog.Errorf("Failed to remove pod logs directory %q: %v", name, err)

20.            }

21.        }

22.    }

23.    // Remove dead container log symlinks.

24.    // TODO(random-liu): Remove this after cluster logging supports CRI container log path.

25.    logSymlinks, _ := osInterface.Glob(filepath.Join(legacyContainerLogsDir, fmt.Sprintf("*.%s", legacyLogSuffix)))

26.    for _, logSymlink := range logSymlinks {

27.        if _, err := osInterface.Stat(logSymlink); os.IsNotExist(err) {

28.            err := osInterface.Remove(logSymlink)

29.            if err != nil {

30.                glog.Errorf("Failed to remove container log dead symlink %q: %v", logSymlink, err)

31.            }

32.        }

33.    }

34.    return nil

35. }

首先会读取/var/log/pods目录下面的子目录，下面的目录名称都是pod的uid，如果pod已经删除那么直接把pod所属的目录删除，然后删除/var/log/containers目录下的软连接。

至此单次ContainerGC的流程结束。

k8s为了回收机器上的存储资源同时有ImageGC对image资源进行回收

ImageGC同样定义了image gc manager和gc policy。

1. //pkg/kubelet/image/image_gc_manager.go

2. type ImageGCPolicy struct {

3.    // Any usage above this threshold will always trigger garbage collection.

4.    // This is the highest usage we will allow.

5.    HighThresholdPercent int

6.    // Any usage below this threshold will never trigger garbage collection.

7.    // This is the lowest threshold we will try to garbage collect to.

8.    LowThresholdPercent int

9.    // Minimum age at which an image can be garbage collected.

10.    MinAge time.Duration

11. }

12. type realImageGCManager struct {

13.    // Container runtime

14.    runtime container.Runtime

15.    // Records of images and their use.

16.    imageRecords     map[string]*imageRecord

17.    imageRecordsLock sync.Mutex

18.    // The image garbage collection policy in use.

19.    policy ImageGCPolicy

20.    // cAdvisor instance.

21.    cadvisor cadvisor.Interface

22.    // Recorder for Kubernetes events.

23.    recorder record.EventRecorder

24.    // Reference to this node.

25.    nodeRef *v1.ObjectReference

26.    // Track initialization

27.    initialized bool

28.    // imageCache is the cache of latest image list.

29.    imageCache imageCache

30. }

策略也主要有三个参数:

• HighThresholdPercent 高于此阈值将进行回收

• LowThresholdPercent 低于此阈值将不会触发回收

• MinAge 回收image的最小年龄

在这个文件里同样有一个GarbageCollect方法

1. func (im *realImageGCManager) detectImages(detectTime time.Time) error {

2.    images, err := im.runtime.ListImages()

3.    if err != nil {

4.        return err

5.    }

6.    pods, err := im.runtime.GetPods(true)

7.    if err != nil {

8.        return err

9.    }

10.    // Make a set of images in use by containers.

11.    imagesInUse := sets.NewString()

12.    for _, pod := range pods {

13.        for _, container := range pod.Containers {

14.            glog.V(5).Infof("Pod %s/%s, container %s uses image %s(%s)", pod.Namespace, pod.Name, container.Name, container.Image, container.ImageID)

15.            imagesInUse.Insert(container.ImageID)

16.        }

17.    }

18.    // Add new images and record those being used.

19.    now := time.Now()

20.    currentImages := sets.NewString()

21.    im.imageRecordsLock.Lock()

22.    defer im.imageRecordsLock.Unlock()

23.    for _, image := range images {

24.        glog.V(5).Infof("Adding image ID %s to currentImages", image.ID)

25.        currentImages.Insert(image.ID)

26.        // New image, set it as detected now.

27.        if _, ok := im.imageRecords[image.ID]; !ok {

28.            glog.V(5).Infof("Image ID %s is new", image.ID)

29.            im.imageRecords[image.ID] = &imageRecord{

30.                firstDetected: detectTime,

31.            }

32.        }

33.        // Set last used time to now if the image is being used.

34.        if isImageUsed(image, imagesInUse) {

35.            glog.V(5).Infof("Setting Image ID %s lastUsed to %v", image.ID, now)

36.            im.imageRecords[image.ID].lastUsed = now

37.        }

38.        glog.V(5).Infof("Image ID %s has size %d", image.ID, image.Size)

39.        im.imageRecords[image.ID].size = image.Size

40.    }

41.    // Remove old images from our records.

42.    for image := range im.imageRecords {

43.        if !currentImages.Has(image) {

44.            glog.V(5).Infof("Image ID %s is no longer present; removing from imageRecords", image)

45.            delete(im.imageRecords, image)

46.        }

47.    }

48.    return nil

49. }

50. func (im *realImageGCManager) GarbageCollect() error {

51.    // Get disk usage on disk holding images.

52.    fsInfo, err := im.cadvisor.ImagesFsInfo()

53.    if err != nil {

54.        return err

55.    }

56.    capacity := int64(fsInfo.Capacity)

57.    available := int64(fsInfo.Available)

58.    if available > capacity {

59.        glog.Warningf("available %d is larger than capacity %d", available, capacity)

60.        available = capacity

61.    }

62.    // Check valid capacity.

63.    if capacity == 0 {

64.        err := fmt.Errorf("invalid capacity %d on device %q at mount point %q", capacity, fsInfo.Device, fsInfo.Mountpoint)

65.        im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.InvalidDiskCapacity, err.Error())

66.        return err

67.    }

68.    // If over the max threshold, free enough to place us at the lower threshold.

69.    usagePercent := 100 - int(available*100/capacity)

70.    if usagePercent >= im.policy.HighThresholdPercent {

71.        amountToFree := capacity*int64(100-im.policy.LowThresholdPercent)/100 - available

72.        glog.Infof("[imageGCManager]: Disk usage on %q (%s) is at %d%% which is over the high threshold (%d%%). Trying to free %d bytes", fsInfo.Device, fsInfo.Mountpoint, usagePercent, im.policy.HighThresholdPercent, amountToFree)

73.        freed, err := im.freeSpace(amountToFree, time.Now())

74.        if err != nil {

75.            return err

76.        }

77.        if freed < amountToFree {

78.            err := fmt.Errorf("failed to garbage collect required amount of images. Wanted to free %d bytes, but freed %d bytes", amountToFree, freed)

79.            im.recorder.Eventf(im.nodeRef, v1.EventTypeWarning, events.FreeDiskSpaceFailed, err.Error())

80.            return err

81.        }

82.    }

83.    return nil

84. }

85. func (im *realImageGCManager) DeleteUnusedImages() (int64, error) {

86.    return im.freeSpace(math.MaxInt64, time.Now())

87. }

88. // Tries to free bytesToFree worth of images on the disk.

89. //

90. // Returns the number of bytes free and an error if any occurred. The number of

91. // bytes freed is always returned.

92. // Note that error may be nil and the number of bytes free may be less

93. // than bytesToFree.

94. func (im *realImageGCManager) freeSpace(bytesToFree int64, freeTime time.Time) (int64, error) {

95.    err := im.detectImages(freeTime)

96.    if err != nil {

97.        return 0, err

98.    }

99.    im.imageRecordsLock.Lock()

100.    defer im.imageRecordsLock.Unlock()

101.    // Get all images in eviction order.

102.    images := make([]evictionInfo, 0, len(im.imageRecords))

103.    for image, record := range im.imageRecords {

104.        images = append(images, evictionInfo{

105.            id:          image,

106.            imageRecord: *record,

107.        })

108.    }

109.    sort.Sort(byLastUsedAndDetected(images))

110.    // Delete unused images until we've freed up enough space.

111.    var deletionErrors []error

112.    spaceFreed := int64(0)

113.    for _, image := range images {

114.        glog.V(5).Infof("Evaluating image ID %s for possible garbage collection", image.id)

115.        // Images that are currently in used were given a newer lastUsed.

116.        if image.lastUsed.Equal(freeTime) || image.lastUsed.After(freeTime) {

117.            glog.V(5).Infof("Image ID %s has lastUsed=%v which is >= freeTime=%v, not eligible for garbage collection", image.id, image.lastUsed, freeTime)

118.            break

119.        }

120.        // Avoid garbage collect the image if the image is not old enough.

121.        // In such a case, the image may have just been pulled down, and will be used by a container right away.

122.        if freeTime.Sub(image.firstDetected) < im.policy.MinAge {

123.            glog.V(5).Infof("Image ID %s has age %v which is less than the policy's minAge of %v, not eligible for garbage collection", image.id, freeTime.Sub(image.firstDetected), im.policy.MinAge)

124.            continue

125.        }

126.        // Remove image. Continue despite errors.

127.        glog.Infof("[imageGCManager]: Removing image %q to free %d bytes", image.id, image.size)

128.        err := im.runtime.RemoveImage(container.ImageSpec{Image: image.id})

129.        if err != nil {

130.            deletionErrors = append(deletionErrors, err)

131.            continue

132.        }

133.        delete(im.imageRecords, image.id)

134.        spaceFreed += image.size

135.        if spaceFreed >= bytesToFree {

136.            break

137.        }

138.    }

139.    if len(deletionErrors) > 0 {

140.        return spaceFreed, fmt.Errorf("wanted to free %d bytes, but freed %d bytes space with errors in image deletion: %v", bytesToFree, spaceFreed, errors.NewAggregate(deletionErrors))

141.    }

142.    return spaceFreed, nil

143. }

GarbageCollect方法也是在pkg/kubelet/kubelet.go里面进行调用，每调用一次执行一次回收流程，每次调用间隔是5分钟。 GarbageCollect首先会调用cadvisor获取监控机器上的资源总量可用量，当发现使用的资源已经大于设置的最高阈值将会触发回收行为，并且计算出需要回收的空间大小。

freeSpace方法则是真正的回收过程，过程分两步。

• 探测机器上的image

• 删除老的image

探测镜像: 

探测机器上的image是获取机器上的image列表并如果是正在使用的image标记最后使用时间和image大小和第一次探测到的时间，并把这个列表的image放到imageRecords里面进行缓存，imageRecords是个map结结构的数据类型。

回收镜像： 

回收镜像首先遍历imageRecords里面的镜像，放到一个数组里面排序，按照最后使用时间或者第一次探测到的时间进行排序。 下一步则对数组里面的镜像进行回收，如果镜像还在使用中或者最后探测到的时间小于设置的MinAge则不进行回收，否则删除镜像。回收过程中如果发现现在机器上的资源已经小于设置的LowThresholdPercent那么跳出回收流程。

原文：

https://www.jianshu.com/p/a6a6f6bab4a1

https://www.jianshu.com/p/2531c043cd70

---END---

K8S培训推荐

Kubernetes线下实战培训，采用3+1新的培训模式（3天线下实战培训，1年内可免费再次参加），资深一线讲师，实操环境实践，现场答疑互动，培训内容覆盖：Docker架构、镜像、数据存储、网络、以及最佳实践。Kubernetes实战内容，Kubernetes设计、Pod、常用对象操作，Kuberentes调度系统、QoS、Helm、网络、存储、CI/CD、日志监控等。<了解更多详情>

北京站、上海站：1月4-6日； 咨询/报名：曹辉/15999647409