From e336aff91964be4b5b5cd1d3302c41acecba6e55 Mon Sep 17 00:00:00 2001 From: "Ma,Qian" Date: Thu, 12 May 2022 12:46:16 +0800 Subject: [PATCH 1/4] Update k8sCleaner.go Try to use kubectl to cleanup_k8s --- .../gcp/pkg/stub/cleaner/k8sCleaner.go | 240 +++++++++++------- 1 file changed, 149 insertions(+), 91 deletions(-) diff --git a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go index ced40fa20..02513bc4b 100644 --- a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go +++ b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go @@ -1,21 +1,22 @@ package cleaner import ( - "fmt" - "time" - - "github.com/sirupsen/logrus" - appV1 "k8s.io/api/apps/v1" - batchV1 "k8s.io/api/batch/v1" - apiCoreV1 "k8s.io/api/core/v1" - apiExtV1Beta1 "k8s.io/api/extensions/v1beta1" - "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - typedAppV1 "k8s.io/client-go/kubernetes/typed/apps/v1" - typedBatchV1 "k8s.io/client-go/kubernetes/typed/batch/v1" - corev1 "k8s.io/client-go/kubernetes/typed/core/v1" - "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" - _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "fmt" + "os/exec" + "time" + + "github.com/sirupsen/logrus" + appV1 "k8s.io/api/apps/v1" + batchV1 "k8s.io/api/batch/v1" + apiCoreV1 "k8s.io/api/core/v1" + apiExtV1Beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + typedAppV1 "k8s.io/client-go/kubernetes/typed/apps/v1" + typedBatchV1 "k8s.io/client-go/kubernetes/typed/batch/v1" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/kubernetes/typed/extensions/v1beta1" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ) // Attempts to remove all objects in a kubernetes cluster connected to persistent resources. @@ -44,7 +45,7 @@ func NewK8sCleaner(cs kubernetes.Interface, log *logrus.Entry) *clusterCleaner { func (cc *clusterCleaner) Cleanup() (bool, error) { cc.log.Debug("Attempt to clean up cluster") - if isClean, err := cc.cleanAllNamespaces(cc.clientSet); err != nil { + if isClean, err := cc.cleanAllIngressInNamespace(cc.clientSet); err != nil { cc.log.Error("couldn't clean all namespaces: ", err.Error()) return false, err } else if !isClean { // only cleanup pods, pvc, and pv if all stateful sets, deployments, ... are gone @@ -74,62 +75,99 @@ func (cc *clusterCleaner) Cleanup() (bool, error) { return isClean, err } -func (cc *clusterCleaner) cleanPvcsInAllNamespaces(clientSet kubernetes.Interface) (bool, error) { - namespaces, err := cc.nsIf.List(v1.ListOptions{}) +func (cc *clusterCleaner) cleanServiceInAllNamespaces(clientSet kubernetes.Interface) (bool, error) { + cmd := exec.Command("bash", "-c", "kubectl delete service"+"--all") + + out, err := cmd.Output() + if err != nil { - cc.log.Error("couldn't enlist all namespaces. err: ", err) + log.Errorf("Could not delete service in all namespace: %v, %v", err, out) return false, err } - outChan := make(chan *helperResultStruct, len(namespaces.Items)) - for _, ns := range namespaces.Items { - if ns.GetName() == v1.NamespaceSystem { - continue + return true, nil +} + +type helperResultStruct struct { + isClean bool + err error +} + +func (cc *clusterCleaner) cleanPvcsInAllNamespaces(clientSet kubernetes.Interface) (bool, error) { + /* namespaces, err := cc.nsIf.List(v1.ListOptions{}) + if err != nil { + cc.log.Error("couldn't enlist all namespaces. err: ", err) + return false, err } - go func(nsName string, pvcIf corev1.PersistentVolumeClaimInterface, out chan *helperResultStruct) { - isClean, err := cc.cleanAllPvcInNamespace(nsName, pvcIf) - out <- &helperResultStruct{isClean, err} - }(ns.GetName(), clientSet.CoreV1().PersistentVolumeClaims(ns.GetName()), outChan) - } - numExpectedResults := len(namespaces.Items) - 1 - allClean, err := cc.collectResults(time.Minute, numExpectedResults, outChan) + outChan := make(chan *helperResultStruct, len(namespaces.Items)) + for _, ns := range namespaces.Items { + if ns.GetName() == v1.NamespaceSystem { + continue + } + go func(nsName string, pvcIf corev1.PersistentVolumeClaimInterface, out chan *helperResultStruct) { + isClean, err := cc.cleanAllPvcInNamespace(nsName, pvcIf) + out <- &helperResultStruct{isClean, err} + }(ns.GetName(), clientSet.CoreV1().PersistentVolumeClaims(ns.GetName()), outChan) + } - if err != nil { - return false, err - } + numExpectedResults := len(namespaces.Items) - 1 + allClean, err := cc.collectResults(time.Minute, numExpectedResults, outChan) - return allClean, nil + if err != nil { + return false, err + } -} + return allClean, nil*/ + cmd := exec.Command("bash", "-c", "kubectl delete pvc"+"--all") + + out, err := cmd.Output() -func (cc *clusterCleaner) cleanPodsInAllNamespaces(clientSet kubernetes.Interface) (bool, error) { - namespaces, err := cc.nsIf.List(v1.ListOptions{}) if err != nil { - cc.log.Error("couldn't enlist all namespaces. err: ", err) + log.Errorf("Could not delete pvc in all namespace: %v, %v", err, out) return false, err } - outChan := make(chan *helperResultStruct, len(namespaces.Items)) - for _, ns := range namespaces.Items { - if ns.GetName() == v1.NamespaceSystem { - continue - } - go func(nsName string, podIf corev1.PodInterface, out chan *helperResultStruct) { - isClean, err := cc.cleanAllPodsInNamespace(nsName, podIf) - out <- &helperResultStruct{isClean, err} - }(ns.GetName(), clientSet.CoreV1().Pods(ns.GetName()), outChan) - } + return true, nil - numExpectedResults := len(namespaces.Items) - 1 - allClean, err := cc.collectResults(time.Minute, numExpectedResults, outChan) +} + +func (cc *clusterCleaner) cleanPodsInAllNamespaces(clientSet kubernetes.Interface) (bool, error) { + /*namespaces, err := cc.nsIf.List(v1.ListOptions{}) + if err != nil { + cc.log.Error("couldn't enlist all namespaces. err: ", err) + return false, err + } + + outChan := make(chan *helperResultStruct, len(namespaces.Items)) + for _, ns := range namespaces.Items { + if ns.GetName() == v1.NamespaceSystem { + continue + } + go func(nsName string, podIf corev1.PodInterface, out chan *helperResultStruct) { + isClean, err := cc.cleanAllPodsInNamespace(nsName, podIf) + out <- &helperResultStruct{isClean, err} + }(ns.GetName(), clientSet.CoreV1().Pods(ns.GetName()), outChan) + } + + numExpectedResults := len(namespaces.Items) - 1 + allClean, err := cc.collectResults(time.Minute, numExpectedResults, outChan) + + if err != nil { + return false, err + } + + return allClean, nil*/ + cmd := exec.Command("bash", "-c", "kubectl delete pod"+"--all") + + out, err := cmd.Output() if err != nil { + log.Errorf("Could not delete pod in all namespace: %v, %v", err, out) return false, err } - return allClean, nil - + return true, nil } type helperResultStruct struct { @@ -353,30 +391,40 @@ func (cc *clusterCleaner) enableStatefulSetForceDeleteIfNecessary(statefulSet *a return nil } -func (cc *clusterCleaner) cleanAllIngressInNamespace(ns string, ingIf v1beta1.IngressInterface) (bool, error) { - list, err := ingIf.List(v1.ListOptions{IncludeUninitialized: true}) - if err != nil { - cc.log.Errorf("couldn't list all ingresses in the namespace %s. err: %s", ns, err.Error()) - return false, err - } - if len(list.Items) == 0 { - return true, nil - } +func (cc *clusterCleaner) cleanAllIngressInNamespace(clientSet kubernetes.Interface) (bool, error) { + /* list, err := ingIf.List(v1.ListOptions{IncludeUninitialized: true}) + if err != nil { + cc.log.Errorf("couldn't list all ingresses in the namespace %s. err: %s", ns, err.Error()) + return false, err + } + if len(list.Items) == 0 { + return true, nil + } - now := time.Now() - for i := range list.Items { - if err := cc.enableIngressForceDeleteIfNecessary(&list.Items[i], now, ns, ingIf); err != nil { + now := time.Now() + for i := range list.Items { + if err := cc.enableIngressForceDeleteIfNecessary(&list.Items[i], now, ns, ingIf); err != nil { + return false, err + } + } + + delPol := v1.DeletePropagationForeground + if err := ingIf.DeleteCollection(&v1.DeleteOptions{PropagationPolicy: &delPol}, v1.ListOptions{IncludeUninitialized: true}); err != nil { + cc.log.Error("couldn't delete all ingresses. err: ", err) return false, err } - } - delPol := v1.DeletePropagationForeground - if err := ingIf.DeleteCollection(&v1.DeleteOptions{PropagationPolicy: &delPol}, v1.ListOptions{IncludeUninitialized: true}); err != nil { - cc.log.Error("couldn't delete all ingresses. err: ", err) + return false, nil*/ + cmd := exec.Command("bash", "-c", "kubectl delete ingress"+"--all") + + out, err := cmd.Output() + + if err != nil { + log.Errorf("Could not list outdated clusters: %v, %v", err, out) return false, err } - return false, nil + return true, nil } func (cc *clusterCleaner) enableIngressForceDeleteIfNecessary(ingress *apiExtV1Beta1.Ingress, now time.Time, ns string, ingIf v1beta1.IngressInterface) error { @@ -594,36 +642,46 @@ func (cc *clusterCleaner) enablePodForceDeleteIfNecessary(pod *apiCoreV1.Pod, no } func (cc *clusterCleaner) deletePersistentVolumes(pvIf corev1.PersistentVolumeInterface) (bool, error) { - cc.log.Debug("clean up all pv") + /*cc.log.Debug("clean up all pv") - list, err := pvIf.List(v1.ListOptions{IncludeUninitialized: true}) - if err != nil { - cc.log.Error("couldn't list all persistent volume claims. err: ", err) - return false, err - } + list, err := pvIf.List(v1.ListOptions{IncludeUninitialized: true}) + if err != nil { + cc.log.Error("couldn't list all persistent volume claims. err: ", err) + return false, err + } - if len(list.Items) == 0 { - cc.log.Debugf("no pvs are left") - return true, nil - } + if len(list.Items) == 0 { + cc.log.Debugf("no pvs are left") + return true, nil + } - cc.log.Debugf("remove %d pvs", len(list.Items)) + cc.log.Debugf("remove %d pvs", len(list.Items)) - // we try a force-delete -> remove finalizers if existent - now := time.Now() - for i := range list.Items { - if err := cc.enablePVForceDeleteIfNecessary(&list.Items[i], now, pvIf); err != nil { - return false, err - } - } + // we try a force-delete -> remove finalizers if existent + now := time.Now() + for i := range list.Items { + if err := cc.enablePVForceDeleteIfNecessary(&list.Items[i], now, pvIf); err != nil { + return false, err + } + } - delPol := v1.DeletePropagationForeground - if err := pvIf.DeleteCollection(&v1.DeleteOptions{PropagationPolicy: &delPol}, v1.ListOptions{IncludeUninitialized: true}); err != nil { - cc.log.Error("couldn't delete all persistent volumes. err: ", err) + delPol := v1.DeletePropagationForeground + if err := pvIf.DeleteCollection(&v1.DeleteOptions{PropagationPolicy: &delPol}, v1.ListOptions{IncludeUninitialized: true}); err != nil { + cc.log.Error("couldn't delete all persistent volumes. err: ", err) + return false, err + } + + return false, nil*/ + cmd := exec.Command("bash", "-c", "kubectl delete pv"+"--all") + + out, err := cmd.Output() + + if err != nil { + log.Errorf("Could not delete pv in all namespace: %v, %v", err, out) return false, err } - return false, nil + return true, nil } func (cc *clusterCleaner) enablePVForceDeleteIfNecessary(pv *apiCoreV1.PersistentVolume, now time.Time, pvIf corev1.PersistentVolumeInterface) error { From a05d6c522d473118437084a2eb26b1aa30fef0b3 Mon Sep 17 00:00:00 2001 From: "Ma,Qian" Date: Thu, 12 May 2022 13:49:04 +0800 Subject: [PATCH 2/4] Update k8sCleaner.go --- src/services/gcp/pkg/stub/cleaner/k8sCleaner.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go index 02513bc4b..07c81505b 100644 --- a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go +++ b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go @@ -81,7 +81,7 @@ func (cc *clusterCleaner) cleanServiceInAllNamespaces(clientSet kubernetes.Inter out, err := cmd.Output() if err != nil { - log.Errorf("Could not delete service in all namespace: %v, %v", err, out) + cc.log.Errorf("Could not delete service in all namespace: %v, %v", err, out) return false, err } @@ -124,7 +124,7 @@ func (cc *clusterCleaner) cleanPvcsInAllNamespaces(clientSet kubernetes.Interfac out, err := cmd.Output() if err != nil { - log.Errorf("Could not delete pvc in all namespace: %v, %v", err, out) + cc.log.Errorf("Could not delete pvc in all namespace: %v, %v", err, out) return false, err } @@ -163,18 +163,13 @@ func (cc *clusterCleaner) cleanPodsInAllNamespaces(clientSet kubernetes.Interfac out, err := cmd.Output() if err != nil { - log.Errorf("Could not delete pod in all namespace: %v, %v", err, out) + cc.log.Errorf("Could not delete pod in all namespace: %v, %v", err, out) return false, err } return true, nil } -type helperResultStruct struct { - isClean bool - err error -} - func (cc *clusterCleaner) cleanAllNamespaces(clientSet kubernetes.Interface) (bool, error) { namespaces, err := cc.nsIf.List(v1.ListOptions{}) if err != nil { @@ -420,7 +415,7 @@ func (cc *clusterCleaner) cleanAllIngressInNamespace(clientSet kubernetes.Interf out, err := cmd.Output() if err != nil { - log.Errorf("Could not list outdated clusters: %v, %v", err, out) + cc.log.Errorf("Could not list outdated clusters: %v, %v", err, out) return false, err } @@ -677,7 +672,7 @@ func (cc *clusterCleaner) deletePersistentVolumes(pvIf corev1.PersistentVolumeIn out, err := cmd.Output() if err != nil { - log.Errorf("Could not delete pv in all namespace: %v, %v", err, out) + cc.log.Errorf("Could not delete pv in all namespace: %v, %v", err, out) return false, err } From 6dfa78b32fcb4ceda21b104771119f4d0c813bcf Mon Sep 17 00:00:00 2001 From: "Ma,Qian" Date: Thu, 12 May 2022 14:37:28 +0800 Subject: [PATCH 3/4] Update k8sCleaner.go --- src/services/gcp/pkg/stub/cleaner/k8sCleaner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go index 07c81505b..691a99615 100644 --- a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go +++ b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go @@ -242,10 +242,10 @@ func (cc *clusterCleaner) cleanupNamespace(ns string, results := make(chan *helperResultStruct, 8) - go func() { + /*go func() { ingressClean, ingErr := cc.cleanAllIngressInNamespace(ns, ingIf) results <- &helperResultStruct{ingressClean, ingErr} - }() + }()*/ go func() { deploymentClean, podErr := cc.cleanAllStatefulSetInNamespace(ns, statefulSetIf) @@ -672,7 +672,7 @@ func (cc *clusterCleaner) deletePersistentVolumes(pvIf corev1.PersistentVolumeIn out, err := cmd.Output() if err != nil { - cc.log.Errorf("Could not delete pv in all namespace: %v, %v", err, out) + cc.log.Error("Could not delete pv in all namespace: %v, %v", err, out) return false, err } From 1fa9214f4b2014ee62346f9ce0c941a6ad6eefe3 Mon Sep 17 00:00:00 2001 From: "Ma,Qian" Date: Fri, 13 May 2022 08:52:36 +0800 Subject: [PATCH 4/4] Update k8sCleaner.go --- .../gcp/pkg/stub/cleaner/k8sCleaner.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go index 691a99615..b7b199bff 100644 --- a/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go +++ b/src/services/gcp/pkg/stub/cleaner/k8sCleaner.go @@ -46,30 +46,30 @@ func (cc *clusterCleaner) Cleanup() (bool, error) { cc.log.Debug("Attempt to clean up cluster") if isClean, err := cc.cleanAllIngressInNamespace(cc.clientSet); err != nil { - cc.log.Error("couldn't clean all namespaces: ", err.Error()) - return false, err + cc.log.Error("couldn't clean ingress: ", err.Error()) + //return false, err } else if !isClean { // only cleanup pods, pvc, and pv if all stateful sets, deployments, ... are gone - return false, nil + //return false, nil } if isClean, err := cc.cleanPodsInAllNamespaces(cc.clientSet); err != nil { cc.log.Error("couldn't remove all pods: ", err.Error()) - return false, nil + //return false, nil } else if !isClean { // only cleanup pvc after all pods are gone - return false, nil + //return false, nil } if isClean, err := cc.cleanPvcsInAllNamespaces(cc.clientSet); err != nil { cc.log.Error("couldn't remove all persistent volume claims: ", err.Error()) - return false, nil + //return false, nil } else if !isClean { // only cleanup pv after all claims are gone - return false, nil + //return false, nil } isClean, err := cc.deletePersistentVolumes(cc.pvIf) if err != nil { cc.log.Error("couldn't remove all persistent volumes: ", err.Error()) - return false, err + //return false, err } return isClean, err @@ -415,11 +415,11 @@ func (cc *clusterCleaner) cleanAllIngressInNamespace(clientSet kubernetes.Interf out, err := cmd.Output() if err != nil { - cc.log.Errorf("Could not list outdated clusters: %v, %v", err, out) - return false, err + cc.log.Errorf("Could not clean ingress in all-namespaces: %v, %v", err, out) + //return false, err } - return true, nil + return true, err } func (cc *clusterCleaner) enableIngressForceDeleteIfNecessary(ingress *apiExtV1Beta1.Ingress, now time.Time, ns string, ingIf v1beta1.IngressInterface) error {