......@@ -19,7 +19,7 @@
dependencies:
- setup-test-env
services:
- docker:stable-dind
- docker:19.03.0-dind
variables:
NODE_ENV: "production"
RAILS_ENV: "production"
......
......
......@@ -35,7 +35,7 @@
<<: *review-base
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-qa-alpine
services:
- docker:stable-dind
- docker:19.03.0-dind
tags:
- gitlab-org
- docker
......@@ -261,6 +261,7 @@ danger-review:
except:
refs:
- master
- /^[\d-]+-stable(-ee)?$/
variables:
- $CI_COMMIT_REF_NAME =~ /^ce-to-ee-.*/
- $CI_COMMIT_REF_NAME =~ /.*-stable(-ee)?-prepare-.*/
......
......
......@@ -127,6 +127,7 @@
.section-header ~ .section.line {
margin-left: $gl-padding;
display: block;
}
}
......
......
......@@ -752,7 +752,7 @@ class MergeRequest < ApplicationRecord
end
def check_mergeability
MergeRequests::MergeabilityCheckService.new(self).execute
MergeRequests::MergeabilityCheckService.new(self).execute(retry_lease: false)
end
# rubocop: enable CodeReuse/ServiceClass
......
......
......@@ -3,6 +3,7 @@
module MergeRequests
class MergeabilityCheckService < ::BaseService
include Gitlab::Utils::StrongMemoize
include Gitlab::ExclusiveLeaseHelpers
delegate :project, to: :@merge_request
delegate :repository, to: :project
......@@ -21,13 +22,35 @@ module MergeRequests
# where we need the current state of the merge ref in repository, the `recheck`
# argument is required.
#
# retry_lease - Concurrent calls wait for at least 10 seconds until the
# lease is granted (other process finishes running). Returns an error
# ServiceResponse if the lease is not granted during this time.
#
# Returns a ServiceResponse indicating merge_status is/became can_be_merged
# and the merge-ref is synced. Success in case of being/becoming mergeable,
# error otherwise.
def execute(recheck: false)
def execute(recheck: false, retry_lease: true)
return ServiceResponse.error(message: 'Invalid argument') unless merge_request
return ServiceResponse.error(message: 'Unsupported operation') if Gitlab::Database.read_only?
return check_mergeability(recheck) unless merge_ref_auto_sync_lock_enabled?
in_write_lock(retry_lease: retry_lease) do |retried|
# When multiple calls are waiting for the same lock (retry_lease),
# it's possible that when granted, the MR status was already updated for
# that object, therefore we reset if there was a lease retry.
merge_request.reset if retried
check_mergeability(recheck)
end
rescue FailedToObtainLockError => error
ServiceResponse.error(message: error.message)
end
private
attr_reader :merge_request
def check_mergeability(recheck)
recheck! if recheck
update_merge_status
......@@ -46,9 +69,21 @@ module MergeRequests
ServiceResponse.success(payload: payload)
end
private
# It's possible for this service to send concurrent requests to Gitaly in order
# to "git update-ref" the same ref. Therefore we handle a light exclusive
# lease here.
#
def in_write_lock(retry_lease:, &block)
lease_key = "mergeability_check:#{merge_request.id}"
attr_reader :merge_request
lease_opts = {
ttl: 1.minute,
retries: retry_lease ? 10 : 0,
sleep_sec: retry_lease ? 1.second : 0
}
in_lock(lease_key, lease_opts, &block)
end
def payload
strong_memoize(:payload) do
......@@ -116,5 +151,9 @@ module MergeRequests
def merge_ref_auto_sync_enabled?
Feature.enabled?(:merge_ref_auto_sync, project, default_enabled: true)
end
def merge_ref_auto_sync_lock_enabled?
Feature.enabled?(:merge_ref_auto_sync_lock, project, default_enabled: true)
end
end
end
......@@ -89,4 +89,4 @@
%span.icon-wrapper.pipeline-status
= render 'ci/status/icon', status: project.commit.last_pipeline.detailed_status(current_user), type: 'commit', tooltip_placement: 'top', path: pipeline_path
.updated-note
%span Updated #{updated_tooltip}
%span #{_('Updated')} #{updated_tooltip}
---
title: Properly translate term in projects list
merge_request: 30958
author:
type: fixed
---
title: Improve job log rendering performance
merge_request: 31262
author:
type: performance
---
title: Add exclusive lease to mergeability check process
merge_request: 31082
author:
type: fixed
---
title: Fix Docker in Docker (DIND) listen port behavior change by adding DOCKER_TLS_CERTDIR in CI job templates.
merge_request: 31201
author: Cameron Boulton
type: fixed
......@@ -65,28 +65,24 @@ Below you can read more about how to use it and how does it work.
Currently, we are using _multi-project pipeline_-like approach to run QA
pipelines.
![QA on merge requests CI/CD architecture](../img/qa_on_merge_requests_cicd_architecture.png)
<details>
<summary>Show mermaid source</summary>
<pre>
```mermaid
graph LR
A1 -.->|1. Triggers an omnibus-gitlab pipeline and wait for it to be done| A2
B2[<b>`Trigger-qa` stage</b><br />`Trigger:qa-test` job] -.->|2. Triggers a gitlab-qa pipeline and wait for it to be done| A3
B2[`Trigger-qa` stage<br>`Trigger:qa-test` job] -.->|2. Triggers a gitlab-qa pipeline and wait for it to be done| A3
subgraph gitlab-ce/ee pipeline
A1[<b>`test` stage</b><br />`package-and-qa` job]
subgraph "gitlab-ce/ee pipeline"
A1[`test` stage<br>`package-and-qa` job]
end
subgraph omnibus-gitlab pipeline
A2[<b>`Trigger-docker` stage</b><br />`Trigger:gitlab-docker` job] -->|once done| B2
subgraph "omnibus-gitlab pipeline"
A2[`Trigger-docker` stage<br>`Trigger:gitlab-docker` job] -->|once done| B2
end
subgraph gitlab-qa pipeline
A3>QA jobs run] -.->|3. Reports back the pipeline result to the `package-and-qa` job<br />and post the result on the original commit tested| A1
subgraph "gitlab-qa pipeline"
A3>QA jobs run] -.->|3. Reports back the pipeline result to the `package-and-qa` job<br>and post the result on the original commit tested| A1
end
</pre>
</details>
```
1. Developer triggers a manual action, that can be found in CE / EE merge
requests. This starts a chain of pipelines in multiple projects.
......
......
doc/development/testing_guide/img/qa_on_merge_requests_cicd_architecture.png

63.3 KiB

doc/development/testing_guide/img/review_apps_cicd_architecture.png

133 KiB

......@@ -8,38 +8,33 @@ Review Apps are automatically deployed by each pipeline, both in
### CI/CD architecture diagram
![Review Apps CI/CD architecture](img/review_apps_cicd_architecture.png)
<details>
<summary>Show mermaid source</summary>
<pre>
```mermaid
graph TD
build-qa-image -.->|once the `prepare` stage is done| gitlab:assets:compile
review-build-cng -->|triggers a CNG-mirror pipeline and wait for it to be done| CNG-mirror
review-build-cng -.->|once the `test` stage is done| review-deploy
review-deploy -.->|once the `review` stage is done| review-qa-smoke
subgraph 1. gitlab-ce/ee `prepare` stage
subgraph "1. gitlab-ce/ee `prepare` stage"
build-qa-image
end
subgraph 2. gitlab-ce/ee `test` stage
subgraph "2. gitlab-ce/ee `test` stage"
gitlab:assets:compile -->|plays dependent job once done| review-build-cng
end
subgraph 3. gitlab-ce/ee `review` stage
review-deploy["review-deploy<br /><br />Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br /><br />Cloud Native images are deployed to the `review-apps-ce` or `review-apps-ee`<br />Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."]
subgraph "3. gitlab-ce/ee `review` stage"
review-deploy["review-deploy<br><br>Helm deploys the Review App using the Cloud<br/>Native images built by the CNG-mirror pipeline.<br><br>Cloud Native images are deployed to the `review-apps-ce` or `review-apps-ee`<br>Kubernetes (GKE) cluster, in the GCP `gitlab-review-apps` project."]
end
subgraph 4. gitlab-ce/ee `qa` stage
review-qa-smoke[review-qa-smoke<br /><br />gitlab-qa runs the smoke suite against the Review App.]
subgraph "4. gitlab-ce/ee `qa` stage"
review-qa-smoke[review-qa-smoke<br><br>gitlab-qa runs the smoke suite against the Review App.]
end
subgraph CNG-mirror pipeline
subgraph "CNG-mirror pipeline"
CNG-mirror>Cloud Native images are built];
end
</pre>
</details>
```
### Detailed explanation
......
......
......@@ -185,6 +185,49 @@ graph TD;
C-->D;
```
#### Subgraphs
NOTE: **Note:** GitLab 12.1 and up now [requires quotes around subgraph
titles that contain multiple words](https://github.com/knsv/mermaid/pull/845).
Subgraphs can also be included:
~~~
```mermaid
graph TB
SubGraph1 --> SubGraph1Flow
subgraph "SubGraph 1 Flow"
SubGraph1Flow(SubNode 1)
SubGraph1Flow -- Choice1 --> DoChoice1
SubGraph1Flow -- Choice2 --> DoChoice2
end
subgraph "Main Graph"
Node1[Node 1] --> Node2[Node 2]
Node2 --> SubGraph1[Jump to SubGraph1]
SubGraph1 --> FinalThing[Final Thing]
end
```
~~~
```mermaid
graph TB
SubGraph1 --> SubGraph1Flow
subgraph "SubGraph 1 Flow"
SubGraph1Flow(SubNode 1)
SubGraph1Flow -- Choice1 --> DoChoice1
SubGraph1Flow -- Choice2 --> DoChoice2
end
subgraph "Main Graph"
Node1[Node 1] --> Node2[Node 2]
Node2 --> SubGraph1[Jump to SubGraph1]
SubGraph1 --> FinalThing[Final Thing]
end
```
### Emoji
> If this is not rendered correctly, [view it in GitLab itself](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/user/markdown.md#emoji).
......
......
......@@ -194,16 +194,10 @@ module Gitlab
end
def handle_new_line
css_classes = []
if @sections.any?
css_classes = %w[section line] + sections.map { |section| "s_#{section}" }
end
write_in_tag %{<br/>}
write_raw %{<span class="#{css_classes.join(' ')}"></span>} if css_classes.any?
close_open_tags if @sections.any? && @lineno_in_section == 0
@lineno_in_section += 1
open_new_tag
end
def handle_section(scanner)
......@@ -310,11 +304,24 @@ module Gitlab
if @sections.any?
css_classes << "section"
css_classes << "js-section-header section-header" if @lineno_in_section == 0
css_classes << if @lineno_in_section == 0
"js-section-header section-header"
else
"line"
end
css_classes += sections.map { |section| "js-s-#{section}" }
end
@out << %{<span class="#{css_classes.join(' ')}">}
close_open_tags
@out << if css_classes.any?
%{<span class="#{css_classes.join(' ')}">}
else
%{<span>}
end
@n_open_tags += 1
end
......
......
......@@ -5,6 +5,7 @@ container_scanning:
image: docker:stable
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
# Defining two new variables based on GitLab's CI/CD predefined variables
# https://docs.gitlab.com/ee/ci/variables/#predefined-environment-variables
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
......
......
......@@ -15,17 +15,18 @@ module Gitlab
raise ArgumentError, 'Key needs to be specified' unless key
lease = Gitlab::ExclusiveLease.new(key, timeout: ttl)
retried = false
until uuid = lease.try_obtain
# Keep trying until we obtain the lease. To prevent hammering Redis too
# much we'll wait for a bit.
sleep(sleep_sec)
break if (retries -= 1) < 0
(retries -= 1) < 0 ? break : retried ||= true
end
raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid
yield
yield(retried)
ensure
Gitlab::ExclusiveLease.cancel(key, uuid)
end
......
......
......@@ -546,7 +546,7 @@ describe Projects::JobsController, :clean_gitlab_redis_shared_state do
expect(response).to have_gitlab_http_status(:ok)
expect(json_response['id']).to eq job.id
expect(json_response['status']).to eq job.status
expect(json_response['html']).to eq('<span class="">BUILD TRACE</span>')
expect(json_response['html']).to eq('<span>BUILD TRACE</span>')
end
end
......
......
......@@ -4,11 +4,11 @@ describe Gitlab::Ci::Ansi2html do
subject { described_class }
it "prints non-ansi as-is" do
expect(convert_html("Hello")).to eq('<span class="">Hello</span>')
expect(convert_html("Hello")).to eq('<span>Hello</span>')
end
it "strips non-color-changing control sequences" do
expect(convert_html("Hello \e[2Kworld")).to eq('<span class="">Hello world</span>')
expect(convert_html("Hello \e[2Kworld")).to eq('<span>Hello world</span>')
end
it "prints simply red" do
......@@ -32,7 +32,7 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets colors after red on blue" do
expect(convert_html("\e[31;44mHello\e[0m world")).to eq('<span class="term-fg-red term-bg-blue">Hello</span><span class=""> world</span>')
expect(convert_html("\e[31;44mHello\e[0m world")).to eq('<span class="term-fg-red term-bg-blue">Hello</span><span> world</span>')
end
it "performs color change from red/blue to yellow/blue" do
......@@ -44,11 +44,11 @@ describe Gitlab::Ci::Ansi2html do
end
it "performs color change from red/blue to reset to yellow/green" do
expect(convert_html("\e[31;44mHello\e[0m \e[33;42mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello</span><span class=""> </span><span class="term-fg-yellow term-bg-green">world</span>')
expect(convert_html("\e[31;44mHello\e[0m \e[33;42mworld")).to eq('<span class="term-fg-red term-bg-blue">Hello</span><span> </span><span class="term-fg-yellow term-bg-green">world</span>')
end
it "ignores unsupported codes" do
expect(convert_html("\e[51mHello\e[0m")).to eq('<span class="">Hello</span>')
expect(convert_html("\e[51mHello\e[0m")).to eq('<span>Hello</span>')
end
it "prints light red" do
......@@ -72,8 +72,8 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets bold text" do
expect(convert_html("\e[1mHello\e[21m world")).to eq('<span class="term-bold">Hello</span><span class=""> world</span>')
expect(convert_html("\e[1mHello\e[22m world")).to eq('<span class="term-bold">Hello</span><span class=""> world</span>')
expect(convert_html("\e[1mHello\e[21m world")).to eq('<span class="term-bold">Hello</span><span> world</span>')
expect(convert_html("\e[1mHello\e[22m world")).to eq('<span class="term-bold">Hello</span><span> world</span>')
end
it "prints italic text" do
......@@ -81,7 +81,7 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets italic text" do
expect(convert_html("\e[3mHello\e[23m world")).to eq('<span class="term-italic">Hello</span><span class=""> world</span>')
expect(convert_html("\e[3mHello\e[23m world")).to eq('<span class="term-italic">Hello</span><span> world</span>')
end
it "prints underlined text" do
......@@ -89,7 +89,7 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets underlined text" do
expect(convert_html("\e[4mHello\e[24m world")).to eq('<span class="term-underline">Hello</span><span class=""> world</span>')
expect(convert_html("\e[4mHello\e[24m world")).to eq('<span class="term-underline">Hello</span><span> world</span>')
end
it "prints concealed text" do
......@@ -97,7 +97,7 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets concealed text" do
expect(convert_html("\e[8mHello\e[28m world")).to eq('<span class="term-conceal">Hello</span><span class=""> world</span>')
expect(convert_html("\e[8mHello\e[28m world")).to eq('<span class="term-conceal">Hello</span><span> world</span>')
end
it "prints crossed-out text" do
......@@ -105,7 +105,7 @@ describe Gitlab::Ci::Ansi2html do
end
it "resets crossed-out text" do
expect(convert_html("\e[9mHello\e[29m world")).to eq('<span class="term-cross">Hello</span><span class=""> world</span>')
expect(convert_html("\e[9mHello\e[29m world")).to eq('<span class="term-cross">Hello</span><span> world</span>')
end
it "can print 256 xterm fg colors" do
......@@ -137,15 +137,15 @@ describe Gitlab::Ci::Ansi2html do
end
it "prints &lt;" do
expect(convert_html("<")).to eq('<span class="">&lt;</span>')
expect(convert_html("<")).to eq('<span>&lt;</span>')
end
it "replaces newlines with line break tags" do
expect(convert_html("\n")).to eq('<span class=""><br/><span class=""></span></span>')
expect(convert_html("\n")).to eq('<span><br/></span>')
end
it "groups carriage returns with newlines" do
expect(convert_html("\r\n")).to eq('<span class=""><br/><span class=""></span></span>')
expect(convert_html("\r\n")).to eq('<span><br/></span>')
end
describe "incremental update" do
......@@ -182,7 +182,7 @@ describe Gitlab::Ci::Ansi2html do
context "with partial sequence" do
let(:pre_text) { "Hello\e" }
let(:pre_html) { "<span class=\"\">Hello</span>" }
let(:pre_html) { "<span>Hello</span>" }
let(:text) { "[1m World" }
let(:html) { "<span class=\"term-bold\"> World</span>" }
......@@ -191,9 +191,9 @@ describe Gitlab::Ci::Ansi2html do
context 'with new line' do
let(:pre_text) { "Hello\r" }
let(:pre_html) { "<span class=\"\">Hello\r</span>" }
let(:pre_html) { "<span>Hello\r</span>" }
let(:text) { "\nWorld" }
let(:html) { "<span class=\"\"><br/><span class=\"\">World</span></span>" }
let(:html) { "<span><br/>World</span>" }
it_behaves_like 'stateable converter'
end
......@@ -220,7 +220,7 @@ describe Gitlab::Ci::Ansi2html do
text = "#{section_start}Some text#{section_end}"
class_name_start = section_start.gsub("\033[0K", '').gsub('<', '&lt;')
class_name_end = section_end.gsub("\033[0K", '').gsub('<', '&lt;')
html = %{<span class="">#{class_name_start}Some text#{class_name_end}</span>}
html = %{<span>#{class_name_start}Some text#{class_name_end}</span>}
expect(convert_html(text)).to eq(html)
end
......@@ -230,12 +230,11 @@ describe Gitlab::Ci::Ansi2html do
let(:text) { "#{section_start}Some text#{section_end}" }
it 'prints light red' do
text = "#{section_start}\e[91mHello\e[0m\n#{section_end}"
text = "#{section_start}\e[91mHello\e[0m\nLine 1\nLine 2\nLine 3\n#{section_end}"
header = %{<span class="term-fg-l-red section js-section-header section-header js-s-#{class_name(section_name)}">Hello</span>}
line_break = %{<span class="section js-section-header section-header js-s-#{class_name(section_name)}"><br/></span>}
line = %{<span class="section line s_#{class_name(section_name)}"></span>}
empty_line = %{<span class="section js-s-#{class_name(section_name)}"></span>}
html = "#{section_start_html}#{header}#{line_break}#{line}#{empty_line}#{section_end_html}"
output_line = %{<span class="section line js-s-#{class_name(section_name)}">Line 1<br/>Line 2<br/>Line 3<br/></span>}
html = "#{section_start_html}#{header}#{line_break}#{output_line}#{section_end_html}"
expect(convert_html(text)).to eq(html)
end
......
......