diff --git a/test/integration/controller/jobset_controller_test.go b/test/integration/controller/jobset_controller_test.go index 7502afec..7e2f788f 100644 --- a/test/integration/controller/jobset_controller_test.go +++ b/test/integration/controller/jobset_controller_test.go @@ -1672,6 +1672,315 @@ var _ = ginkgo.Describe("JobSet controller", func() { }, }, }), + ginkgo.Entry("DependsOn: rjob-b depends on ready status of rjob-a", &testCase{ + makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { + return testing.MakeJobSet("depends-on", ns.Name). + SuccessPolicy(&jobset.SuccessPolicy{Operator: jobset.OperatorAll, TargetReplicatedJobs: []string{}}). + ReplicatedJob(testing.MakeReplicatedJob("rjob-a"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(1). + Obj()). + ReplicatedJob(testing.MakeReplicatedJob("rjob-b"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(3). + DependsOn([]jobset.DependsOn{ + { + Name: "rjob-a", + Status: jobset.ReadyStatus, + }, + }). + Obj()) + }, + skipCreationCheck: true, + steps: []*step{ + { + // First check. + // Replicated-Job-A should be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + }, + { + // Second check. + // Set the Replicated-Job-A status to ready. + // Replicated-Job-B depends on ready status of Replicated-Job-A + jobUpdateFn: func(jobList *batchv1.JobList) { + readyReplicatedJob(jobList, "rjob-a") + }, + // Replicated-Job-A must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + }, + { + Name: "rjob-a", + Ready: 1, + }, + }) + }, + }, + { + // Final check. + // Update the Replicated-Job-B status to ready. + jobUpdateFn: func(jobList *batchv1.JobList) { + readyReplicatedJob(jobList, "rjob-b") + }, + // All Jobs must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 4 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + // All Jobs must be in the ready status. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + Ready: 3, + }, + { + Name: "rjob-a", + Ready: 1, + }, + }) + }, + }, + }, + }), + ginkgo.Entry("DependsOn: rjob-b depends on complete status of rjob-a, and rjob-c depends on ready status of rjob-b", &testCase{ + makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { + return testing.MakeJobSet("depends-on", ns.Name). + SuccessPolicy(&jobset.SuccessPolicy{Operator: jobset.OperatorAll, TargetReplicatedJobs: []string{}}). + ReplicatedJob(testing.MakeReplicatedJob("rjob-a"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(1). + Obj()). + ReplicatedJob(testing.MakeReplicatedJob("rjob-b"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(1). + DependsOn([]jobset.DependsOn{ + { + Name: "rjob-a", + Status: jobset.CompleteStatus, + }, + }). + Obj()). + ReplicatedJob(testing.MakeReplicatedJob("rjob-c"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(3). + DependsOn([]jobset.DependsOn{ + { + Name: "rjob-b", + Status: jobset.ReadyStatus, + }, + }). + Obj()) + }, + skipCreationCheck: true, + steps: []*step{ + { + // First check. + // Replicated-Job-A should be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + }, + { + // Second check. + // Set the Replicated-Job-A status to complete. + // Replicated-Job-B depends on complete status of Replicated-Job-A + jobUpdateFn: func(jobList *batchv1.JobList) { + completeJob(&jobList.Items[0]) + }, + // Replicated-Job-A must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-c", + }, + { + Name: "rjob-b", + }, + { + Name: "rjob-a", + Succeeded: 1, + }, + }) + }, + }, + { + // Third check. + // Set the Replicated-Job-B status to ready. + // Replicated-Job-C depends on ready status of Replicated-Job-B + jobUpdateFn: func(jobList *batchv1.JobList) { + readyReplicatedJob(jobList, "rjob-b") + }, + // Replicated-Job-A and Replicated-Job-B must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 2 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-c", + }, + { + Name: "rjob-b", + Ready: 1, + }, + { + Name: "rjob-a", + Succeeded: 1, + }, + }) + }, + }, + { + // Final check. + // Complete all Jobs. + jobUpdateFn: completeAllJobs, + // Replicated-Job-A, Replicated-Job-B, and Replicated-Job-C must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 5 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + // All Jobs must be in the succeeded status. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-c", + Succeeded: 3, + }, + { + Name: "rjob-b", + Succeeded: 1, + }, + { + Name: "rjob-a", + Succeeded: 1, + }, + }) + }, + }, + }, + }), + ginkgo.Entry("DependsOn: resume suspended JobSet when rjob-b depends on ready status of rjob-a", &testCase{ + makeJobSet: func(ns *corev1.Namespace) *testing.JobSetWrapper { + return testing.MakeJobSet("depends-on", ns.Name). + SuccessPolicy(&jobset.SuccessPolicy{Operator: jobset.OperatorAll, TargetReplicatedJobs: []string{}}). + ReplicatedJob(testing.MakeReplicatedJob("rjob-a"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(1). + Obj()). + ReplicatedJob(testing.MakeReplicatedJob("rjob-b"). + Job(testing.MakeJobTemplate("job", ns.Name).PodSpec(testing.TestPodSpec).Obj()). + Replicas(3). + DependsOn([]jobset.DependsOn{ + { + Name: "rjob-a", + Status: jobset.ReadyStatus, + }, + }). + Obj()). + Suspend(true) + }, + skipCreationCheck: true, + steps: []*step{ + { + // Ensure that Replicated-Job-A is suspended. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + }, + { + Name: "rjob-a", + Suspended: 1, + }, + }) + }, + }, + { + // Resume the JobSet. + jobSetUpdateFn: func(js *jobset.JobSet) { + suspendJobSet(js, false) + }, + // Only the Replicated-Job-A must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + // Only the Replicated-Job-A should be unsuspended due to DependsOn order. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + }, + { + Name: "rjob-a", + Suspended: 0, + }, + }) + }, + }, + { + // Update the Replicated-Job-A to the ready status. + jobUpdateFn: func(jobList *batchv1.JobList) { + readyReplicatedJob(jobList, "rjob-a") + }, + // Only the Replicated-Job-A must be created. + checkJobCreation: func(js *jobset.JobSet) { + expectedStarts := 1 + gomega.Eventually(testutil.NumJobs, timeout, interval).WithArguments(ctx, k8sClient, js).Should(gomega.Equal(expectedStarts)) + }, + // Replicated-Job-B must be unsuspended. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + Suspended: 0, + }, + { + Name: "rjob-a", + Ready: 1, + }, + }) + }, + }, + { + // Update the Replicated-Job-B to the ready status. + jobUpdateFn: func(jobList *batchv1.JobList) { + readyReplicatedJob(jobList, "rjob-b") + }, + // Replicated-Job-A and Replicated-Job-B must have the correct statuses. + checkJobSetState: func(js *jobset.JobSet) { + matchJobSetReplicatedStatus(js, []jobset.ReplicatedJobStatus{ + { + Name: "rjob-b", + Ready: 3, + Suspended: 0, + }, + { + Name: "rjob-a", + Ready: 1, + Suspended: 0, + }, + }) + }, + }, + }, + }), ) // end of DescribeTable ginkgo.When("A JobSet is managed by another controller", ginkgo.Ordered, func() {