From 419814749d023cdc6a03143487da94f483eea17c Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Fri, 12 Oct 2018 13:52:27 +0200 Subject: [PATCH 1/2] Use another ENV variable for k8s prereq timeout For backwards compatibility, the default value is the same as KUBE_WAIT_FOR_LIVE. But it can be adjusted to whatever is needed. --- plugins/kubernetes/README.md | 4 ++++ .../app/models/kubernetes/deploy_executor.rb | 13 +++++++------ .../test/models/kubernetes/deploy_executor_test.rb | 6 +++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/plugins/kubernetes/README.md b/plugins/kubernetes/README.md index 73eda5fcbd..b0b3fee8f8 100644 --- a/plugins/kubernetes/README.md +++ b/plugins/kubernetes/README.md @@ -125,6 +125,10 @@ Via [Template filler](/plugins/kubernetes/app/models/kubernetes/template_filler. Add a role with only a `Pod`, `metadata.annotations.samson/prerequisite: true`, and command to run a migrations. It will be executed before the rest is deployed. +For default it waits for 10 minutes before timeout, you can change the timeout +using KUBE_WAIT_FOR_PREREQ env variable (specified in seconds). + + ### StatefulSet On kubernetes <1.7 they can only be updated with `OnDelete` updateStrategy, diff --git a/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb b/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb index 72c848a05c..0eed726f6f 100644 --- a/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb +++ b/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb @@ -6,6 +6,7 @@ module Kubernetes class DeployExecutor WAIT_FOR_LIVE = ENV.fetch('KUBE_WAIT_FOR_LIVE', 10).to_i.minutes + WAIT_FOR_PREREQ = ENV.fetch('KUBE_WAIT_FOR_PREREQ', 10).to_i.minutes STABILITY_CHECK_DURATION = 1.minute TICK = 2.seconds RESTARTED = "Restarted" @@ -59,11 +60,11 @@ def execute(*) prerequisites, deploys = @release.release_docs.partition(&:prerequisite?) if prerequisites.any? @output.puts "First deploying prerequisite ..." if deploys.any? - return false unless deploy_and_watch(prerequisites) + return false unless deploy_and_watch(prerequisites, WAIT_FOR_PREREQ) @output.puts "Now deploying other roles ..." if deploys.any? end if deploys.any? - return false unless deploy_and_watch(deploys) + return false unless deploy_and_watch(deploys, WAIT_FOR_LIVE) end true end @@ -72,7 +73,7 @@ def execute(*) # check all pods and see if they are running # once they are running check if they are stable (for apps only, since jobs are finished and will not change) - def wait_for_resources_to_complete(release_docs) + def wait_for_resources_to_complete(release_docs, timeout) raise "prerequisites should not check for stability" if @testing_for_stability @wait_start_time = Time.now @output.puts "Waiting for pods to be created" @@ -100,7 +101,7 @@ def wait_for_resources_to_complete(release_docs) if stopped = not_ready.select(&:stop).presence unstable!('one or more pods stopped', stopped) return statuses - elsif seconds_waiting > WAIT_FOR_LIVE + elsif seconds_waiting > timeout @output.puts "TIMEOUT, pods took too long to get live" return statuses end @@ -388,9 +389,9 @@ def deploy(release_docs) Samson::Parallelizer.map(resources, db: true, &:deploy) end - def deploy_and_watch(release_docs) + def deploy_and_watch(release_docs, timeout) deploy(release_docs) - result = wait_for_resources_to_complete(release_docs) + result = wait_for_resources_to_complete(release_docs, timeout) if result == true if blue_green = release_docs.select(&:blue_green_color).presence finish_blue_green_deployment(blue_green) diff --git a/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb b/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb index b3bafcd2e3..752bdce421 100644 --- a/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb +++ b/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb @@ -728,7 +728,7 @@ def create_previous_successful_release executor.expects(:wait_for_resources_to_complete).returns(true) executor.instance_variable_set(:@release, release) - assert executor.send(:deploy_and_watch, release.release_docs) + assert executor.send(:deploy_and_watch, release.release_docs, 60) out.must_equal <<~OUT Deploying BLUE resources for Pod1 role app-server @@ -757,7 +757,7 @@ def create_previous_successful_release executor.expects(:wait_for_resources_to_complete).returns(true) executor.instance_variable_set(:@release, release) - assert executor.send(:deploy_and_watch, release.release_docs) + assert executor.send(:deploy_and_watch, release.release_docs, 60) out.must_equal <<~OUT Deploying BLUE resources for Pod1 role app-server @@ -781,7 +781,7 @@ def create_previous_successful_release executor.expects(:wait_for_resources_to_complete).returns([]) executor.expects(:print_resource_events) executor.instance_variable_set(:@release, release) - refute executor.send(:deploy_and_watch, release.release_docs) + refute executor.send(:deploy_and_watch, release.release_docs, 60) out.must_equal <<~OUT Deploying BLUE resources for Pod1 role app-server From bad423f2fe3fe9466c35891175afd1d1ebdb1f30 Mon Sep 17 00:00:00 2001 From: Rodrigo Campos Date: Fri, 12 Oct 2018 13:59:44 +0200 Subject: [PATCH 2/2] Add missing documentation for KUBE_WAIT_FOR_LIVE --- plugins/kubernetes/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/kubernetes/README.md b/plugins/kubernetes/README.md index b0b3fee8f8..bc08e4701f 100644 --- a/plugins/kubernetes/README.md +++ b/plugins/kubernetes/README.md @@ -128,6 +128,10 @@ It will be executed before the rest is deployed. For default it waits for 10 minutes before timeout, you can change the timeout using KUBE_WAIT_FOR_PREREQ env variable (specified in seconds). +### Deployment timeouts + +A deploy will wait for 10 minutes for pods to come alive. You can adjust this +timeout using KUBE_WAIT_FOR_LIVE. ### StatefulSet