[jobframework] Handle all errors when job is suspended (#1907)

Author: alexandear

.golangci.yaml

@@ -25,5 +25,6 @@ linters:
     - gocritic
     - goimports
     - govet
+    - loggercheck
     - misspell
     - unconvert

pkg/controller/jobframework/reconciler.go

@@ -389,12 +389,14 @@ func (r *JobReconciler) ReconcileGenericJob(ctx context.Context, req ctrl.Reques
 			if err != nil {
 				log.Error(err, "Unsuspending job")
 				if podset.IsPermanent(err) {
+					allErrs := err
 					// Mark the workload as finished with failure since the is no point to retry.
 					errUpdateStatus := workload.UpdateStatus(ctx, r.client, wl, kueue.WorkloadFinished, metav1.ConditionTrue, FailedToStartFinishedReason, err.Error(), constants.JobControllerName)
 					if errUpdateStatus != nil {
-						log.Error(errUpdateStatus, "Updating workload status, on start failure %s", err.Error())
+						allErrs = errors.Join(err, errUpdateStatus)
+						log.Error(allErrs, "Could not mark Workload as finished after start failure")
 					}
-					return ctrl.Result{}, errUpdateStatus
+					return ctrl.Result{}, allErrs
 				}
 			}
 			return ctrl.Result{}, err

pkg/controller/jobs/job/job_controller_test.go

@@ -768,6 +768,7 @@ func TestReconciler(t *testing.T) {
 					}).
 					Obj(),
 			},
+			wantErr: podset.ErrInvalidPodSetUpdate,
 		},
 		"when workload is admitted and PodSetUpdates conflict between admission checks on annotations, the workload is finished with failure": {
 			job: *baseJobWrapper.Clone().
@@ -839,6 +840,7 @@ func TestReconciler(t *testing.T) {
 					}).
 					Obj(),
 			},
+			wantErr: podset.ErrInvalidPodSetUpdate,
 		},
 		"when workload is admitted and PodSetUpdates conflict between admission checks on nodeSelector, the workload is finished with failure": {
 			job: *baseJobWrapper.Clone().
@@ -910,6 +912,7 @@ func TestReconciler(t *testing.T) {
 					}).
 					Obj(),
 			},
+			wantErr: podset.ErrInvalidPodSetUpdate,
 		},
 		"when workload is admitted and PodSetUpdates conflict between admission check nodeSelector and current node selector, the workload is finished with failure": {
 			job: *baseJobWrapper.Clone().
@@ -959,6 +962,7 @@ func TestReconciler(t *testing.T) {
 					}).
 					Obj(),
 			},
+			wantErr: podset.ErrInvalidPodSetUpdate,
 		},
 		"when workload is admitted the PodSetUpdates values matching for key": {
 			job: *baseJobWrapper.Clone().