...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
} }
} }
.ci-status-icon-preparing,
.ci-status-icon-running { .ci-status-icon-running {
svg { svg {
fill: $blue-400; fill: $blue-400;
... ...
......
...@@ -166,6 +166,7 @@ ...@@ -166,6 +166,7 @@
float: left; float: left;
.accept-merge-request { .accept-merge-request {
&.ci-preparing,
&.ci-pending, &.ci-pending,
&.ci-running { &.ci-running {
@include btn-blue; @include btn-blue;
... ...
......
...@@ -795,6 +795,7 @@ ...@@ -795,6 +795,7 @@
@include mini-pipeline-graph-color($white, $orange-100, $orange-200, $orange-500, $orange-600, $orange-700); @include mini-pipeline-graph-color($white, $orange-100, $orange-200, $orange-500, $orange-600, $orange-700);
} }
&.ci-status-icon-preparing,
&.ci-status-icon-running { &.ci-status-icon-running {
@include mini-pipeline-graph-color($white, $blue-100, $blue-200, $blue-500, $blue-600, $blue-700); @include mini-pipeline-graph-color($white, $blue-100, $blue-200, $blue-500, $blue-600, $blue-700);
} }
... ...
......
...@@ -44,6 +44,7 @@ ...@@ -44,6 +44,7 @@
} }
&.ci-info, &.ci-info,
&.ci-preparing,
&.ci-running { &.ci-running {
@include status-color($blue-100, $blue-500, $blue-600); @include status-color($blue-100, $blue-500, $blue-600);
} }
... ...
......
...@@ -172,6 +172,10 @@ module Ci ...@@ -172,6 +172,10 @@ module Ci
end end
state_machine :status do state_machine :status do
event :enqueue do
transition [:created, :skipped, :manual, :scheduled] => :preparing, if: :any_unmet_prerequisites?
end
event :actionize do event :actionize do
transition created: :manual transition created: :manual
end end
...@@ -185,8 +189,12 @@ module Ci ...@@ -185,8 +189,12 @@ module Ci
end end
event :enqueue_scheduled do event :enqueue_scheduled do
transition scheduled: :preparing, if: ->(build) do
build.scheduled_at&.past? && build.any_unmet_prerequisites?
end
transition scheduled: :pending, if: ->(build) do transition scheduled: :pending, if: ->(build) do
build.scheduled_at && build.scheduled_at < Time.now build.scheduled_at&.past? && !build.any_unmet_prerequisites?
end end
end end
...@@ -204,6 +212,12 @@ module Ci ...@@ -204,6 +212,12 @@ module Ci
end end
end end
after_transition any => [:preparing] do |build|
build.run_after_commit do
Ci::BuildPrepareWorker.perform_async(id)
end
end
after_transition any => [:pending] do |build| after_transition any => [:pending] do |build|
build.run_after_commit do build.run_after_commit do
BuildQueueWorker.perform_async(id) BuildQueueWorker.perform_async(id)
...@@ -355,6 +369,16 @@ module Ci ...@@ -355,6 +369,16 @@ module Ci
!retried? !retried?
end end
def any_unmet_prerequisites?
return false unless Feature.enabled?(:ci_preparing_state, default_enabled: true)
prerequisites.present?
end
def prerequisites
Gitlab::Ci::Build::Prerequisite::Factory.new(self).unmet
end
def expanded_environment_name def expanded_environment_name
return unless has_environment? return unless has_environment?
... ...
......
...@@ -82,10 +82,14 @@ module Ci ...@@ -82,10 +82,14 @@ module Ci
state_machine :status, initial: :created do state_machine :status, initial: :created do
event :enqueue do event :enqueue do
transition [:created, :skipped, :scheduled] => :pending transition [:created, :preparing, :skipped, :scheduled] => :pending
transition [:success, :failed, :canceled] => :running transition [:success, :failed, :canceled] => :running
end end
event :prepare do
transition any - [:preparing] => :preparing
end
event :run do event :run do
transition any - [:running] => :running transition any - [:running] => :running
end end
...@@ -118,7 +122,7 @@ module Ci ...@@ -118,7 +122,7 @@ module Ci
# Do not add any operations to this state_machine # Do not add any operations to this state_machine
# Create a separate worker for each new operation # Create a separate worker for each new operation
before_transition [:created, :pending] => :running do |pipeline| before_transition [:created, :preparing, :pending] => :running do |pipeline|
pipeline.started_at = Time.now pipeline.started_at = Time.now
end end
...@@ -141,7 +145,7 @@ module Ci ...@@ -141,7 +145,7 @@ module Ci
end end
end end
after_transition [:created, :pending] => :running do |pipeline| after_transition [:created, :preparing, :pending] => :running do |pipeline|
pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) } pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
end end
...@@ -149,7 +153,7 @@ module Ci ...@@ -149,7 +153,7 @@ module Ci
pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) } pipeline.run_after_commit { PipelineMetricsWorker.perform_async(pipeline.id) }
end end
after_transition [:created, :pending, :running] => :success do |pipeline| after_transition [:created, :preparing, :pending, :running] => :success do |pipeline|
pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) } pipeline.run_after_commit { PipelineSuccessWorker.perform_async(pipeline.id) }
end end
...@@ -597,6 +601,7 @@ module Ci ...@@ -597,6 +601,7 @@ module Ci
retry_optimistic_lock(self) do retry_optimistic_lock(self) do
case latest_builds_status.to_s case latest_builds_status.to_s
when 'created' then nil when 'created' then nil
when 'preparing' then prepare
when 'pending' then enqueue when 'pending' then enqueue
when 'running' then run when 'running' then run
when 'success' then succeed when 'success' then succeed
... ...
......
...@@ -39,10 +39,14 @@ module Ci ...@@ -39,10 +39,14 @@ module Ci
state_machine :status, initial: :created do state_machine :status, initial: :created do
event :enqueue do event :enqueue do
transition created: :pending transition [:created, :preparing] => :pending
transition [:success, :failed, :canceled, :skipped] => :running transition [:success, :failed, :canceled, :skipped] => :running
end end
event :prepare do
transition any - [:preparing] => :preparing
end
event :run do event :run do
transition any - [:running] => :running transition any - [:running] => :running
end end
...@@ -76,6 +80,7 @@ module Ci ...@@ -76,6 +80,7 @@ module Ci
retry_optimistic_lock(self) do retry_optimistic_lock(self) do
case statuses.latest.status case statuses.latest.status
when 'created' then nil when 'created' then nil
when 'preparing' then prepare
when 'pending' then enqueue when 'pending' then enqueue
when 'running' then run when 'running' then run
when 'success' then succeed when 'success' then succeed
... ...
......
...@@ -66,7 +66,10 @@ class CommitStatus < ActiveRecord::Base ...@@ -66,7 +66,10 @@ class CommitStatus < ActiveRecord::Base
end end
event :enqueue do event :enqueue do
transition [:created, :skipped, :manual, :scheduled] => :pending # A CommitStatus will never have prerequisites, but this event
# is shared by Ci::Build, which cannot progress unless prerequisites
# are satisfied.
transition [:created, :preparing, :skipped, :manual, :scheduled] => :pending, unless: :any_unmet_prerequisites?
end end
event :run do event :run do
...@@ -74,26 +77,26 @@ class CommitStatus < ActiveRecord::Base ...@@ -74,26 +77,26 @@ class CommitStatus < ActiveRecord::Base
end end
event :skip do event :skip do
transition [:created, :pending] => :skipped transition [:created, :preparing, :pending] => :skipped
end end
event :drop do event :drop do
transition [:created, :pending, :running, :scheduled] => :failed transition [:created, :preparing, :pending, :running, :scheduled] => :failed
end end
event :success do event :success do
transition [:created, :pending, :running] => :success transition [:created, :preparing, :pending, :running] => :success
end end
event :cancel do event :cancel do
transition [:created, :pending, :running, :manual, :scheduled] => :canceled transition [:created, :preparing, :pending, :running, :manual, :scheduled] => :canceled
end end
before_transition [:created, :skipped, :manual, :scheduled] => :pending do |commit_status| before_transition [:created, :preparing, :skipped, :manual, :scheduled] => :pending do |commit_status|
commit_status.queued_at = Time.now commit_status.queued_at = Time.now
end end
before_transition [:created, :pending] => :running do |commit_status| before_transition [:created, :preparing, :pending] => :running do |commit_status|
commit_status.started_at = Time.now commit_status.started_at = Time.now
end end
...@@ -180,6 +183,10 @@ class CommitStatus < ActiveRecord::Base ...@@ -180,6 +183,10 @@ class CommitStatus < ActiveRecord::Base
false false
end end
def any_unmet_prerequisites?
false
end
def auto_canceled? def auto_canceled?
canceled? && auto_canceled_by_id? canceled? && auto_canceled_by_id?
end end
... ...
......
...@@ -14,7 +14,8 @@ module CommitStatusEnums ...@@ -14,7 +14,8 @@ module CommitStatusEnums
runner_unsupported: 6, runner_unsupported: 6,
stale_schedule: 7, stale_schedule: 7,
job_execution_timeout: 8, job_execution_timeout: 8,
archived_failure: 9 archived_failure: 9,
unmet_prerequisites: 10
} }
end end
end end
...@@ -5,14 +5,14 @@ module HasStatus ...@@ -5,14 +5,14 @@ module HasStatus
DEFAULT_STATUS = 'created'.freeze DEFAULT_STATUS = 'created'.freeze
BLOCKED_STATUS = %w[manual scheduled].freeze BLOCKED_STATUS = %w[manual scheduled].freeze
AVAILABLE_STATUSES = %w[created pending running success failed canceled skipped manual scheduled].freeze AVAILABLE_STATUSES = %w[created preparing pending running success failed canceled skipped manual scheduled].freeze
STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze STARTED_STATUSES = %w[running success failed skipped manual scheduled].freeze
ACTIVE_STATUSES = %w[pending running].freeze ACTIVE_STATUSES = %w[preparing pending running].freeze
COMPLETED_STATUSES = %w[success failed canceled skipped].freeze COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
ORDERED_STATUSES = %w[failed pending running manual scheduled canceled success skipped created].freeze ORDERED_STATUSES = %w[failed preparing pending running manual scheduled canceled success skipped created].freeze
STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3, STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
failed: 4, canceled: 5, skipped: 6, manual: 7, failed: 4, canceled: 5, skipped: 6, manual: 7,
scheduled: 8 }.freeze scheduled: 8, preparing: 9 }.freeze
UnknownStatusError = Class.new(StandardError) UnknownStatusError = Class.new(StandardError)
...@@ -26,6 +26,7 @@ module HasStatus ...@@ -26,6 +26,7 @@ module HasStatus
success = scope_relevant.success.select('count(*)').to_sql success = scope_relevant.success.select('count(*)').to_sql
manual = scope_relevant.manual.select('count(*)').to_sql manual = scope_relevant.manual.select('count(*)').to_sql
scheduled = scope_relevant.scheduled.select('count(*)').to_sql scheduled = scope_relevant.scheduled.select('count(*)').to_sql
preparing = scope_relevant.preparing.select('count(*)').to_sql
pending = scope_relevant.pending.select('count(*)').to_sql pending = scope_relevant.pending.select('count(*)').to_sql
running = scope_relevant.running.select('count(*)').to_sql running = scope_relevant.running.select('count(*)').to_sql
skipped = scope_relevant.skipped.select('count(*)').to_sql skipped = scope_relevant.skipped.select('count(*)').to_sql
...@@ -37,12 +38,14 @@ module HasStatus ...@@ -37,12 +38,14 @@ module HasStatus
WHEN (#{builds})=(#{skipped}) THEN 'skipped' WHEN (#{builds})=(#{skipped}) THEN 'skipped'
WHEN (#{builds})=(#{success}) THEN 'success' WHEN (#{builds})=(#{success}) THEN 'success'
WHEN (#{builds})=(#{created}) THEN 'created' WHEN (#{builds})=(#{created}) THEN 'created'
WHEN (#{builds})=(#{preparing}) THEN 'preparing'
WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success' WHEN (#{builds})=(#{success})+(#{skipped}) THEN 'success'
WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled' WHEN (#{builds})=(#{success})+(#{skipped})+(#{canceled}) THEN 'canceled'
WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending' WHEN (#{builds})=(#{created})+(#{skipped})+(#{pending}) THEN 'pending'
WHEN (#{running})+(#{pending})>0 THEN 'running' WHEN (#{running})+(#{pending})>0 THEN 'running'
WHEN (#{manual})>0 THEN 'manual' WHEN (#{manual})>0 THEN 'manual'
WHEN (#{scheduled})>0 THEN 'scheduled' WHEN (#{scheduled})>0 THEN 'scheduled'
WHEN (#{preparing})>0 THEN 'preparing'
WHEN (#{created})>0 THEN 'running' WHEN (#{created})>0 THEN 'running'
ELSE 'failed' ELSE 'failed'
END)" END)"
...@@ -70,6 +73,7 @@ module HasStatus ...@@ -70,6 +73,7 @@ module HasStatus
state_machine :status, initial: :created do state_machine :status, initial: :created do
state :created, value: 'created' state :created, value: 'created'
state :preparing, value: 'preparing'
state :pending, value: 'pending' state :pending, value: 'pending'
state :running, value: 'running' state :running, value: 'running'
state :failed, value: 'failed' state :failed, value: 'failed'
...@@ -81,6 +85,7 @@ module HasStatus ...@@ -81,6 +85,7 @@ module HasStatus
end end
scope :created, -> { where(status: 'created') } scope :created, -> { where(status: 'created') }
scope :preparing, -> { where(status: 'preparing') }
scope :relevant, -> { where(status: AVAILABLE_STATUSES - ['created']) } scope :relevant, -> { where(status: AVAILABLE_STATUSES - ['created']) }
scope :running, -> { where(status: 'running') } scope :running, -> { where(status: 'running') }
scope :pending, -> { where(status: 'pending') } scope :pending, -> { where(status: 'pending') }
...@@ -90,14 +95,14 @@ module HasStatus ...@@ -90,14 +95,14 @@ module HasStatus
scope :skipped, -> { where(status: 'skipped') } scope :skipped, -> { where(status: 'skipped') }
scope :manual, -> { where(status: 'manual') } scope :manual, -> { where(status: 'manual') }
scope :scheduled, -> { where(status: 'scheduled') } scope :scheduled, -> { where(status: 'scheduled') }
scope :alive, -> { where(status: [:created, :pending, :running]) } scope :alive, -> { where(status: [:created, :preparing, :pending, :running]) }
scope :created_or_pending, -> { where(status: [:created, :pending]) } scope :created_or_pending, -> { where(status: [:created, :pending]) }
scope :running_or_pending, -> { where(status: [:running, :pending]) } scope :running_or_pending, -> { where(status: [:running, :pending]) }
scope :finished, -> { where(status: [:success, :failed, :canceled]) } scope :finished, -> { where(status: [:success, :failed, :canceled]) }
scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) } scope :failed_or_canceled, -> { where(status: [:failed, :canceled]) }
scope :cancelable, -> do scope :cancelable, -> do
where(status: [:running, :pending, :created, :scheduled]) where(status: [:running, :preparing, :pending, :created, :scheduled])
end end
end end
... ...
......
...@@ -78,6 +78,10 @@ class Deployment < ActiveRecord::Base ...@@ -78,6 +78,10 @@ class Deployment < ActiveRecord::Base
Commit.truncate_sha(sha) Commit.truncate_sha(sha)
end end
def cluster
project.deployment_platform(environment: environment.name)&.cluster
end
def last? def last?
self == environment.last_deployment self == environment.last_deployment
end end
... ...
......
...@@ -11,7 +11,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated ...@@ -11,7 +11,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
runner_unsupported: 'Your runner is outdated, please upgrade your runner', runner_unsupported: 'Your runner is outdated, please upgrade your runner',
stale_schedule: 'Delayed job could not be executed by some reason, please try again', stale_schedule: 'Delayed job could not be executed by some reason, please try again',
job_execution_timeout: 'The script exceeded the maximum execution time set for the job', job_execution_timeout: 'The script exceeded the maximum execution time set for the job',
archived_failure: 'The job is archived and cannot be run' archived_failure: 'The job is archived and cannot be run',
unmet_prerequisites: 'The job failed to complete prerequisite tasks'
}.freeze }.freeze
private_constant :CALLOUT_FAILURE_MESSAGES private_constant :CALLOUT_FAILURE_MESSAGES
... ...
......
# frozen_string_literal: true
module Ci
class PrepareBuildService
attr_reader :build
def initialize(build)
@build = build
end
def execute
prerequisites.each(&:complete!)
unless build.enqueue
build.drop!(:unmet_prerequisites)
end
end
private
def prerequisites
build.prerequisites
end
end
end
...@@ -71,6 +71,7 @@ ...@@ -71,6 +71,7 @@
- pipeline_hooks:build_hooks - pipeline_hooks:build_hooks
- pipeline_hooks:pipeline_hooks - pipeline_hooks:pipeline_hooks
- pipeline_processing:build_finished - pipeline_processing:build_finished
- pipeline_processing:ci_build_prepare
- pipeline_processing:build_queue - pipeline_processing:build_queue
- pipeline_processing:build_success - pipeline_processing:build_success
- pipeline_processing:pipeline_process - pipeline_processing:pipeline_process
... ...
......
# frozen_string_literal: true
module Ci
class BuildPrepareWorker
include ApplicationWorker
include PipelineQueue
queue_namespace :pipeline_processing
def perform(build_id)
Ci::Build.find_by_id(build_id).try do |build|
Ci::PrepareBuildService.new(build).execute
end
end
end
end
...@@ -5,6 +5,8 @@ class ClusterConfigureWorker ...@@ -5,6 +5,8 @@ class ClusterConfigureWorker
include ClusterQueue include ClusterQueue
def perform(cluster_id) def perform(cluster_id)
return if Feature.enabled?(:ci_preparing_state, default_enabled: true)
Clusters::Cluster.find_by_id(cluster_id).try do |cluster| Clusters::Cluster.find_by_id(cluster_id).try do |cluster|
Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster) Clusters::RefreshService.create_or_update_namespaces_for_cluster(cluster)
end end
... ...
......
...@@ -5,6 +5,8 @@ class ClusterProjectConfigureWorker ...@@ -5,6 +5,8 @@ class ClusterProjectConfigureWorker
include ClusterQueue include ClusterQueue
def perform(project_id) def perform(project_id)
return if Feature.enabled?(:ci_preparing_state, default_enabled: true)
project = Project.find(project_id) project = Project.find(project_id)
::Clusters::RefreshService.create_or_update_namespaces_for_project(project) ::Clusters::RefreshService.create_or_update_namespaces_for_project(project)
... ...
......
---
title: Create Kubernetes resources for projects when their deployment jobs run.
merge_request: 25586
author:
type: changed
...@@ -1022,6 +1022,10 @@ planned for a subsequent release. ...@@ -1022,6 +1022,10 @@ planned for a subsequent release.
buildpack](#custom-buildpacks). buildpack](#custom-buildpacks).
- Auto Test may fail because of a mismatch between testing frameworks. In this - Auto Test may fail because of a mismatch between testing frameworks. In this
case, you may need to customize your `.gitlab-ci.yml` with your test commands. case, you may need to customize your `.gitlab-ci.yml` with your test commands.
- Auto Deploy may fail if it is unable to create a Kubernetes namespace and
service account for your project. See the
[troubleshooting failed deployments](../../user/project/clusters/index.md#troubleshooting-failed-deployment-jobs)
section to debug why these resources were not created.
### Disable the banner instance wide ### Disable the banner instance wide
... ...
......
...@@ -556,26 +556,27 @@ NOTE: **NOTE:** ...@@ -556,26 +556,27 @@ NOTE: **NOTE:**
Prior to GitLab 11.5, `KUBE_TOKEN` was the Kubernetes token of the main Prior to GitLab 11.5, `KUBE_TOKEN` was the Kubernetes token of the main
service account of the cluster integration. service account of the cluster integration.
### Troubleshooting missing `KUBECONFIG` or `KUBE_TOKEN` ### Troubleshooting failed deployment jobs
GitLab will create a new service account specifically for your CI builds. The GitLab will create a namespace and service account specifically for your
new service account is created when the cluster is added to the project. deployment jobs. These resources are created just before the deployment
Sometimes there may be errors that cause the service account creation to fail. job starts. Sometimes there may be errors that cause their creation to fail.
In such instances, your build will not be passed the `KUBECONFIG` or In such instances, your job will fail with the message:
`KUBE_TOKEN` variables and, if you are using Auto DevOps, your Auto DevOps
pipelines will no longer trigger a `production` deploy build. You will need to ```The job failed to complete prerequisite tasks```
check the [logs](../../../administration/logs.md) to debug why the service
account creation failed. You will need to check the [logs](../../../administration/logs.md) to debug
why the namespace and service account creation failed.
A common reason for failure is that the token you gave GitLab did not have A common reason for failure is that the token you gave GitLab did not have
[`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) [`cluster-admin`](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles)
privileges as GitLab expects. privileges as GitLab expects.
Another common problem for why these variables are not being passed to your Another common problem is caused by a missing `KUBECONFIG` or `KUBE_TOKEN`.
builds is that they must have a matching To be passed to your job, it must have a matching
[`environment:name`](../../../ci/environments.md#defining-environments). If [`environment:name`](../../../ci/environments.md#defining-environments). If
your build has no `environment:name` set, it will not be passed the Kubernetes your job has no `environment:name` set, it will not be passed the Kubernetes
credentials. credentials.
## Monitoring your Kubernetes cluster **[ULTIMATE]** ## Monitoring your Kubernetes cluster **[ULTIMATE]**
... ...
......