-
Notifications
You must be signed in to change notification settings - Fork 124
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Rechecking pending Pods #195
base: master
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -47,6 +47,7 @@ rules: | |
- pods | ||
verbs: | ||
- list | ||
- get | ||
--- | ||
apiVersion: apps/v1 | ||
kind: DaemonSet | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,6 +4,7 @@ import ( | |
"context" | ||
"fmt" | ||
"net" | ||
"strings" | ||
"time" | ||
|
||
"github.com/k8snetworkplumbingwg/whereabouts/pkg/allocate" | ||
|
@@ -102,19 +103,73 @@ func (rl *ReconcileLooper) findOrphanedIPsPerPool(ipPools []storage.IPPool) erro | |
func (rl ReconcileLooper) isPodAlive(podRef string, ip string) bool { | ||
for livePodRef, livePod := range rl.liveWhereaboutsPods { | ||
if podRef == livePodRef { | ||
livePodIPs := livePod.ips | ||
logging.Debugf( | ||
"pod reference %s matches allocation; Allocation IP: %s; PodIPs: %s", | ||
livePodRef, | ||
ip, | ||
livePodIPs) | ||
_, isFound := livePodIPs[ip] | ||
return isFound || livePod.phase == v1.PodPending | ||
isFound := isIpOnPod(&livePod, podRef, ip) | ||
if !isFound && (livePod.phase == v1.PodPending) { | ||
/* Sometimes pods are still coming up, and may not yet have Multus | ||
* annotation added to it yet. We don't want to check the IPs yet | ||
* so re-fetch the Pod 5x | ||
*/ | ||
podToMatch := &livePod | ||
retries := 0 | ||
|
||
logging.Debugf("Re-fetching Pending Pod: %s IP-to-match: %s", livePodRef, ip) | ||
|
||
for retries < storage.PodRefreshRetries { | ||
retries += 1 | ||
podToMatch = rl.refreshPod(livePodRef) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. would it make sense to instead of retrying now just put the pods in pending state to a list, and after the main loop (that handles all pods that are not pending) iterate all the ones that were in pending state ? I am unsure if I am overthinking this. |
||
if podToMatch == nil { | ||
logging.Debugf("Cleaning up...") | ||
return false | ||
} else if podToMatch.phase != v1.PodPending { | ||
logging.Debugf("Pending Pod is now in phase: %s", podToMatch.phase) | ||
break | ||
} else { | ||
isFound = isIpOnPod(podToMatch, podRef, ip) | ||
// Short-circuit - Pending Pod may have IP now | ||
if isFound { | ||
logging.Debugf("Pod now has IP annotation while in Pending") | ||
return true | ||
} | ||
time.Sleep(time.Duration(500) * time.Millisecond) | ||
} | ||
} | ||
isFound = isIpOnPod(podToMatch, podRef, ip) | ||
} | ||
|
||
return isFound | ||
} | ||
} | ||
return false | ||
} | ||
|
||
func (rl ReconcileLooper) refreshPod(podRef string) *podWrapper { | ||
namespace, podName := splitPodRef(podRef) | ||
if namespace == "" || podName == "" { | ||
logging.Errorf("Invalid podRef format: %s", podRef) | ||
return nil | ||
} | ||
|
||
pod, err := rl.k8sClient.GetPod(namespace, podName) | ||
if err != nil { | ||
logging.Errorf("Failed to refresh Pod %s: %s\n", podRef, err) | ||
return nil | ||
} | ||
|
||
wrappedPod := wrapPod(*pod) | ||
logging.Debugf("Got refreshed pod: %v", wrappedPod) | ||
return wrappedPod | ||
} | ||
|
||
func splitPodRef(podRef string) (string, string) { | ||
namespacedName := strings.Split(podRef, "/") | ||
if len(namespacedName) != 2 { | ||
logging.Errorf("Failed to split podRef %s", podRef) | ||
return "", "" | ||
} | ||
|
||
return namespacedName[0], namespacedName[1] | ||
} | ||
|
||
func composePodRef(pod v1.Pod) string { | ||
return fmt.Sprintf("%s/%s", pod.GetNamespace(), pod.GetName()) | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -117,6 +117,15 @@ func (i *Client) ListPods(ctx context.Context) ([]v1.Pod, error) { | |
return podList.Items, nil | ||
} | ||
|
||
func (i *Client) GetPod(namespace, name string) (*v1.Pod, error) { | ||
pod, err := i.clientSet.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
return pod, nil | ||
Comment on lines
+121
to
+126
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hm, maybe I'm missing something, but why don't you just |
||
} | ||
|
||
func (i *Client) ListOverlappingIPs(ctx context.Context) ([]whereaboutsv1alpha1.OverlappingRangeIPReservation, error) { | ||
overlappingIPsList := whereaboutsv1alpha1.OverlappingRangeIPReservationList{} | ||
if err := i.client.List(ctx, &overlappingIPsList, &client.ListOptions{}); err != nil { | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: could make sense to export this into an helper func.