|
| 1 | +/* |
| 2 | +Copyright 2020 The KubeOne Authors. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package clientutil |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "fmt" |
| 22 | + "time" |
| 23 | + |
| 24 | + "k8c.io/kubeone/pkg/fail" |
| 25 | + "k8c.io/reconciler/pkg/reconciling" |
| 26 | + |
| 27 | + "github.com/sirupsen/logrus" |
| 28 | + corev1 "k8s.io/api/core/v1" |
| 29 | + "k8s.io/apimachinery/pkg/util/wait" |
| 30 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 31 | +) |
| 32 | + |
| 33 | +const ( |
| 34 | + annotationKeyDescription = "description" |
| 35 | + |
| 36 | + // AnnDynamicallyProvisioned is added to a PV that is dynamically provisioned by kubernetes |
| 37 | + // Because the annotation is defined only at k8s.io/kubernetes, copying the content instead of vendoring |
| 38 | + // https://github.com/kubernetes/kubernetes/blob/v1.21.0/pkg/controller/volume/persistentvolume/util/util.go#L65 |
| 39 | + AnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by" |
| 40 | +) |
| 41 | + |
| 42 | +var VolumeResources = []string{"persistentvolumes", "persistentvolumeclaims"} |
| 43 | + |
| 44 | +func CleanupUnretainedVolumes(ctx context.Context, logger logrus.FieldLogger, c client.Client) error { |
| 45 | + // We disable the PV & PVC creation so nothing creates new PV's while we delete them |
| 46 | + logger.Infoln("Creating ValidatingWebhookConfiguration to disable future PV & PVC creation...") |
| 47 | + if err := disablePVCreation(ctx, c); err != nil { |
| 48 | + return fail.KubeClient(err, "failed to disable future PV & PVC creation.") |
| 49 | + } |
| 50 | + |
| 51 | + pvcList, pvList, err := getDynamicallyProvisionedUnretainedPvs(ctx, c) |
| 52 | + if err != nil { |
| 53 | + return err |
| 54 | + } |
| 55 | + |
| 56 | + // Do not attempt to delete any pods when there are no PVs and PVCs |
| 57 | + if (pvcList != nil && pvList != nil) && len(pvcList.Items) == 0 && len(pvList.Items) == 0 { |
| 58 | + return nil |
| 59 | + } |
| 60 | + |
| 61 | + // Delete all Pods that use PVs. We must keep the remaining pods, otherwise |
| 62 | + // we end up in a deadlock when CSI is used |
| 63 | + if err := cleanupPVCUsingPods(ctx, c); err != nil { |
| 64 | + return fail.KubeClient(err, "failed to clean up PV using pod from user cluster.") |
| 65 | + } |
| 66 | + |
| 67 | + // Delete PVC's |
| 68 | + logger.Infoln("Deleting persistent volume claims...") |
| 69 | + for _, pvc := range pvcList.Items { |
| 70 | + if pvc.DeletionTimestamp == nil { |
| 71 | + identifier := fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name) |
| 72 | + logger.Infoln("Deleting PVC...", identifier) |
| 73 | + |
| 74 | + if err := DeleteIfExists(ctx, c, &pvc); err != nil { |
| 75 | + return fail.KubeClient(err, "failed to delete PVC from user cluster.") |
| 76 | + } |
| 77 | + } |
| 78 | + } |
| 79 | + |
| 80 | + return nil |
| 81 | +} |
| 82 | + |
| 83 | +func disablePVCreation(ctx context.Context, c client.Client) error { |
| 84 | + // Prevent re-creation of PVs and PVCs by using an intentionally defunct admissionWebhook |
| 85 | + creatorGetters := []reconciling.NamedValidatingWebhookConfigurationReconcilerFactory{ |
| 86 | + creationPreventingWebhook("", VolumeResources), |
| 87 | + } |
| 88 | + if err := reconciling.ReconcileValidatingWebhookConfigurations(ctx, creatorGetters, "", c); err != nil { |
| 89 | + return fail.KubeClient(err, "failed to create ValidatingWebhookConfiguration to prevent creation of PVs/PVCs.") |
| 90 | + } |
| 91 | + |
| 92 | + return nil |
| 93 | +} |
| 94 | + |
| 95 | +func cleanupPVCUsingPods(ctx context.Context, c client.Client) error { |
| 96 | + podList := &corev1.PodList{} |
| 97 | + if err := c.List(ctx, podList); err != nil { |
| 98 | + return fail.KubeClient(err, "failed to list Pods from user cluster.") |
| 99 | + } |
| 100 | + |
| 101 | + var pvUsingPods []*corev1.Pod |
| 102 | + for idx := range podList.Items { |
| 103 | + pod := &podList.Items[idx] |
| 104 | + if podUsesPV(pod) { |
| 105 | + pvUsingPods = append(pvUsingPods, pod) |
| 106 | + } |
| 107 | + } |
| 108 | + |
| 109 | + for _, pod := range pvUsingPods { |
| 110 | + if pod.DeletionTimestamp == nil { |
| 111 | + if err := DeleteIfExists(ctx, c, pod); err != nil { |
| 112 | + return fail.KubeClient(err, "failed to delete Pod.") |
| 113 | + } |
| 114 | + } |
| 115 | + } |
| 116 | + |
| 117 | + return nil |
| 118 | +} |
| 119 | + |
| 120 | +func podUsesPV(p *corev1.Pod) bool { |
| 121 | + for _, volume := range p.Spec.Volumes { |
| 122 | + if volume.VolumeSource.PersistentVolumeClaim != nil { |
| 123 | + return true |
| 124 | + } |
| 125 | + } |
| 126 | + |
| 127 | + return false |
| 128 | +} |
| 129 | + |
| 130 | +func getDynamicallyProvisionedUnretainedPvs(ctx context.Context, c client.Client) (*corev1.PersistentVolumeClaimList, *corev1.PersistentVolumeList, error) { |
| 131 | + pvcList := &corev1.PersistentVolumeClaimList{} |
| 132 | + if err := c.List(ctx, pvcList); err != nil { |
| 133 | + return nil, nil, fail.KubeClient(err, "failed to list PVCs from user cluster.") |
| 134 | + } |
| 135 | + allPVList := &corev1.PersistentVolumeList{} |
| 136 | + if err := c.List(ctx, allPVList); err != nil { |
| 137 | + return nil, nil, fail.KubeClient(err, "failed to list PVs from user cluster.") |
| 138 | + } |
| 139 | + pvList := &corev1.PersistentVolumeList{} |
| 140 | + for _, pv := range allPVList.Items { |
| 141 | + // Check only dynamically provisioned PVs with delete reclaim policy to verify provisioner has done the cleanup |
| 142 | + // this filters out everything else because we leave those be |
| 143 | + if pv.Annotations[AnnDynamicallyProvisioned] != "" && pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete { |
| 144 | + pvList.Items = append(pvList.Items, pv) |
| 145 | + } |
| 146 | + } |
| 147 | + |
| 148 | + return pvcList, pvList, nil |
| 149 | +} |
| 150 | + |
| 151 | +func WaitCleanUpVolumes(ctx context.Context, logger logrus.FieldLogger, c client.Client) error { |
| 152 | + logger.Infoln("Waiting for all dynamically provisioned and unretained volumes to get deleted...") |
| 153 | + |
| 154 | + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 5*time.Minute, false, func(ctx context.Context) (bool, error) { |
| 155 | + pvcList, pvList, err := getDynamicallyProvisionedUnretainedPvs(ctx, c) |
| 156 | + if err != nil { |
| 157 | + return false, nil |
| 158 | + } |
| 159 | + |
| 160 | + if (pvcList != nil && pvList != nil) && len(pvcList.Items) == 0 && len(pvList.Items) == 0 { |
| 161 | + return true, nil |
| 162 | + } |
| 163 | + |
| 164 | + return false, nil |
| 165 | + }) |
| 166 | +} |
0 commit comments