......@@ -5,7 +5,7 @@ module API
class Discussion < Grape::Entity
expose :id
expose :individual_note?, as: :individual_note
expose :notes, using: Entities::Note
expose :notes, using: Entities::NoteWithGitlabEmployeeBadge
end
end
end
# frozen_string_literal: true
module API
module Entities
class NoteWithGitlabEmployeeBadge < Note
expose :author, using: Entities::UserWithGitlabEmployeeBadge
expose :resolved_by, using: Entities::UserWithGitlabEmployeeBadge, if: ->(note, options) { note.resolvable? }
end
end
end
# frozen_string_literal: true
module API
module Entities
class UserWithGitlabEmployeeBadge < UserBasic
expose :gitlab_employee?, as: :is_gitlab_employee, if: ->(user, options) { ::Feature.enabled?(:gitlab_employee_badge) && user.gitlab_employee? }
end
end
end
......@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS =
%i[junit codequality sast dependency_scanning container_scanning
dast performance license_management license_scanning metrics lsif
dotenv].freeze
dotenv cobertura].freeze
attributes ALLOWED_KEYS
......@@ -35,6 +35,7 @@ module Gitlab
validates :metrics, array_of_strings_or_string: true
validates :lsif, array_of_strings_or_string: true
validates :dotenv, array_of_strings_or_string: true
validates :cobertura, array_of_strings_or_string: true
end
end
......
......
......@@ -9,7 +9,8 @@ module Gitlab
def self.parsers
{
junit: ::Gitlab::Ci::Parsers::Test::Junit
junit: ::Gitlab::Ci::Parsers::Test::Junit,
cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura
}
end
......
......
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Coverage
class Cobertura
CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
def parse!(xml_data, coverage_report)
root = Hash.from_xml(xml_data)
parse_all(root, coverage_report)
rescue Nokogiri::XML::SyntaxError
raise CoberturaParserError, "XML parsing failed"
rescue
raise CoberturaParserError, "Cobertura parsing failed"
end
private
def parse_all(root, coverage_report)
return unless root.present?
root.each do |key, value|
parse_node(key, value, coverage_report)
end
end
def parse_node(key, value, coverage_report)
if key == 'class'
Array.wrap(value).each do |item|
parse_class(item, coverage_report)
end
elsif value.is_a?(Hash)
parse_all(value, coverage_report)
elsif value.is_a?(Array)
value.each do |item|
parse_all(item, coverage_report)
end
end
end
def parse_class(file, coverage_report)
return unless file["filename"].present? && file["lines"].present?
parsed_lines = parse_lines(file["lines"])
coverage_report.add_file(file["filename"], Hash[parsed_lines])
end
def parse_lines(lines)
line_array = Array.wrap(lines["line"])
line_array.map do |line|
# Using `Integer()` here to raise exception on invalid values
[Integer(line["number"]), Integer(line["hits"])]
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Reports
class CoverageReports
attr_reader :files
def initialize
@files = {}
end
def pick(keys)
coverage_files = files.select do |key|
keys.include?(key)
end
{ files: coverage_files }
end
def add_file(name, line_coverage)
if files[name].present?
line_coverage.each { |line, hits| combine_lines(name, line, hits) }
else
files[name] = line_coverage
end
end
private
def combine_lines(name, line, hits)
if files[name][line].present?
files[name][line] += hits
else
files[name][line] = hits
end
end
end
end
end
end
......@@ -49,7 +49,7 @@ module Gitlab
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
@tree_saver ||= LegacyRelationTreeSaver.new
end
end
end
......
......
# frozen_string_literal: true
module Gitlab
module ImportExport
module JSON
class LegacyWriter
include Gitlab::ImportExport::CommandLineUtil
attr_reader :path
def initialize(path)
@path = path
@last_array = nil
@keys = Set.new
mkdir_p(File.dirname(@path))
file.write('{}')
end
def close
@file&.close
@file = nil
end
def set(hash)
hash.each do |key, value|
write(key, value)
end
end
def write(key, value)
raise ArgumentError, "key '#{key}' already written" if @keys.include?(key)
# rewind by one byte, to overwrite '}'
file.pos = file.size - 1
file.write(',') if @keys.any?
file.write(key.to_json)
file.write(':')
file.write(value.to_json)
file.write('}')
@keys.add(key)
@last_array = nil
@last_array_count = nil
end
def append(key, value)
unless @last_array == key
write(key, [])
@last_array = key
@last_array_count = 0
end
# rewind by two bytes, to overwrite ']}'
file.pos = file.size - 2
file.write(',') if @last_array_count > 0
file.write(value.to_json)
file.write(']}')
@last_array_count += 1
end
private
def file
@file ||= File.open(@path, "wb")
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module ImportExport
module JSON
class StreamingSerializer
include Gitlab::ImportExport::CommandLineUtil
BATCH_SIZE = 100
class Raw < String
def to_json(*_args)
to_s
end
end
def initialize(exportable, relations_schema, json_writer)
@exportable = exportable
@relations_schema = relations_schema
@json_writer = json_writer
end
def execute
serialize_root
includes.each do |relation_definition|
serialize_relation(relation_definition)
end
end
private
attr_reader :json_writer, :relations_schema, :exportable
def serialize_root
attributes = exportable.as_json(
relations_schema.merge(include: nil, preloads: nil))
json_writer.set(attributes)
end
def serialize_relation(definition)
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
key, options = definition.first
record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend
if record.is_a?(ActiveRecord::Relation)
serialize_many_relations(key, record, options)
else
serialize_single_relation(key, record, options)
end
end
def serialize_many_relations(key, records, options)
key_preloads = preloads&.dig(key)
records = records.preload(key_preloads) if key_preloads
records.find_each(batch_size: BATCH_SIZE) do |record|
json = Raw.new(record.to_json(options))
json_writer.append(key, json)
end
end
def serialize_single_relation(key, record, options)
json = Raw.new(record.to_json(options))
json_writer.write(key, json)
end
def includes
relations_schema[:include]
end
def preloads
relations_schema[:preload]
end
end
end
end
end
......@@ -2,7 +2,7 @@
module Gitlab
module ImportExport
class RelationTreeSaver
class LegacyRelationTreeSaver
include Gitlab::ImportExport::CommandLineUtil
def serialize(exportable, relations_tree)
......
......
# frozen_string_literal: true
module Gitlab
module ImportExport
module Project
class LegacyTreeSaver
attr_reader :full_path
def initialize(project:, current_user:, shared:, params: {})
@params = params
@project = project
@current_user = current_user
@shared = shared
@full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
true
rescue => e
@shared.error(e)
false
end
private
# Aware that the resulting hash needs to be pure-hash and
# does not include any AR objects anymore, only objects that run `.to_json`
def fix_project_tree(project_tree)
if @params[:description].present?
project_tree['description'] = @params[:description]
end
project_tree['project_members'] += group_members_array
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def group_members_array
group_members.as_json(reader.group_members_tree).each do |group_member|
group_member['source_type'] = 'Project' # Make group members project members of the future import
end
end
def group_members
return [] unless @current_user.can?(:admin_group, @project.group)
# We need `.where.not(user_id: nil)` here otherwise when a group has an
# invitee, it would make the following query return 0 rows since a NULL
# user_id would be present in the subquery
# See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
end
def tree_saver
@tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new
end
end
end
end
end
......@@ -15,52 +15,40 @@ module Gitlab
end
def save
project_tree = tree_saver.serialize(@project, reader.project_tree)
fix_project_tree(project_tree)
tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
json_writer = ImportExport::JSON::LegacyWriter.new(@full_path)
serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer)
serializer.execute
true
rescue => e
@shared.error(e)
false
ensure
json_writer&.close
end
private
# Aware that the resulting hash needs to be pure-hash and
# does not include any AR objects anymore, only objects that run `.to_json`
def fix_project_tree(project_tree)
if @params[:description].present?
project_tree['description'] = @params[:description]
end
project_tree['project_members'] += group_members_array
end
def reader
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
end
def group_members_array
group_members.as_json(reader.group_members_tree).each do |group_member|
group_member['source_type'] = 'Project' # Make group members project members of the future import
end
def exportable
@project.present(exportable_params)
end
def group_members
return [] unless @current_user.can?(:admin_group, @project.group)
# We need `.where.not(user_id: nil)` here otherwise when a group has an
# invitee, it would make the following query return 0 rows since a NULL
# user_id would be present in the subquery
# See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
def exportable_params
params = {
presenter_class: presenter_class,
current_user: @current_user
}
params[:override_description] = @params[:description] if @params[:description].present?
params
end
def tree_saver
@tree_saver ||= RelationTreeSaver.new
def presenter_class
Projects::ImportExport::ProjectExportPresenter
end
end
end
......
......
......@@ -19,7 +19,6 @@ namespace :gitlab do
if ENV['EXPORT_DEBUG'].present?
ActiveRecord::Base.logger = logger
Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
logger.level = Logger::DEBUG
else
logger.level = Logger::INFO
......
......
......@@ -23,7 +23,6 @@ namespace :gitlab do
if ENV['IMPORT_DEBUG'].present?
ActiveRecord::Base.logger = logger
Gitlab::Metrics::Exporter::SidekiqExporter.instance.start
logger.level = Logger::DEBUG
else
logger.level = Logger::INFO
......
......
......@@ -33,6 +33,6 @@ namespace :sidekiq do
task :launchd do
deprecation_warning!
system(*%w(bin/background_jobs start_no_deamonize))
system(*%w(bin/background_jobs start_silent))
end
end
......@@ -1810,6 +1810,9 @@ msgstr ""
msgid "An error occurred while enabling Service Desk."
msgstr ""
 
msgid "An error occurred while fetching coverage reports."
msgstr ""
msgid "An error occurred while fetching environments."
msgstr ""
 
......@@ -4822,6 +4825,9 @@ msgstr ""
msgid "ClusterIntegration|You must have an RBAC-enabled cluster to install Knative."
msgstr ""
 
msgid "ClusterIntegration|You should select at least two subnets"
msgstr ""
msgid "ClusterIntegration|Your account must have %{link_to_kubernetes_engine}"
msgstr ""
 
......@@ -13325,6 +13331,9 @@ msgstr ""
msgid "No template"
msgstr ""
 
msgid "No test coverage"
msgstr ""
msgid "No thanks, don't show this again"
msgstr ""
 
......@@ -19472,6 +19481,11 @@ msgstr ""
msgid "Test coverage parsing"
msgstr ""
 
msgid "Test coverage: %d hit"
msgid_plural "Test coverage: %d hits"
msgstr[0] ""
msgstr[1] ""
msgid "Test failed."
msgstr ""
 
......
......
......@@ -984,6 +984,136 @@ describe Projects::MergeRequestsController do
end
end
describe 'GET coverage_reports' do
let(:merge_request) do
create(:merge_request,
:with_merge_request_pipeline,
target_project: project,
source_project: project)
end
let(:pipeline) do
create(:ci_pipeline,
:success,
project: merge_request.source_project,
ref: merge_request.source_branch,
sha: merge_request.diff_head_sha)
end
before do
allow_any_instance_of(MergeRequest)
.to receive(:find_coverage_reports)
.and_return(report)
allow_any_instance_of(MergeRequest)
.to receive(:actual_head_pipeline)
.and_return(pipeline)
end
subject do
get :coverage_reports, params: {
namespace_id: project.namespace.to_param,
project_id: project,
id: merge_request.iid
},
format: :json
end
describe 'permissions on a public project with private CI/CD' do
let(:project) { create :project, :repository, :public, :builds_private }
let(:report) { { status: :parsed, data: [] } }
context 'while signed out' do
before do
sign_out(user)
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(:not_found)
expect(response.body).to be_blank
end
end
context 'while signed in as an unrelated user' do
before do
sign_in(create(:user))
end
it 'responds with a 404' do
subject
expect(response).to have_gitlab_http_status(:not_found)
expect(response.body).to be_blank
end
end
end
context 'when pipeline has jobs with coverage reports' do
before do
allow_any_instance_of(MergeRequest)
.to receive(:has_coverage_reports?)
.and_return(true)
end
context 'when processing coverage reports is in progress' do
let(:report) { { status: :parsing } }
it 'sends polling interval' do
expect(Gitlab::PollingInterval).to receive(:set_header)
subject
end
it 'returns 204 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:no_content)
end
end
context 'when processing coverage reports is completed' do
let(:report) { { status: :parsed, data: pipeline.coverage_reports } }
it 'returns coverage reports' do
subject
expect(response).to have_gitlab_http_status(:ok)
expect(json_response).to eq({ 'files' => {} })
end
end
context 'when user created corrupted coverage reports' do
let(:report) { { status: :error, status_reason: 'Failed to parse coverage reports' } }
it 'does not send polling interval' do
expect(Gitlab::PollingInterval).not_to receive(:set_header)
subject
end
it 'returns 400 HTTP status' do
subject
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response).to eq({ 'status_reason' => 'Failed to parse coverage reports' })
end
end
end
context 'when pipeline does not have jobs with coverage reports' do
let(:report) { double }
it 'returns no content' do
subject
expect(response).to have_gitlab_http_status(:no_content)
expect(response.body).to be_empty
end
end
end
describe 'GET test_reports' do
let(:merge_request) do
create(:merge_request,
......
......
......@@ -311,6 +311,12 @@ FactoryBot.define do
end
end
trait :coverage_reports do
after(:build) do |build|
build.job_artifacts << create(:ci_job_artifact, :cobertura, job: build)
end
end
trait :expired do
artifacts_expire_at { 1.minute.ago }
end
......@@ -355,6 +361,8 @@ FactoryBot.define do
options { {} }
end
# TODO: move Security traits to ee_ci_build
# https://gitlab.com/gitlab-org/gitlab/-/issues/210486
trait :dast do
options do
{
......@@ -395,6 +403,14 @@ FactoryBot.define do
end
end
trait :license_scanning do
options do
{
artifacts: { reports: { license_management: 'gl-license-scanning-report.json' } }
}
end
end
trait :non_playable do
status { 'created' }
self.when { 'manual' }
......
......
......@@ -129,6 +129,36 @@ FactoryBot.define do
end
end
trait :cobertura do
file_type { :cobertura }
file_format { :gzip }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/cobertura/coverage.xml.gz'), 'application/x-gzip')
end
end
trait :coverage_gocov_xml do
file_type { :cobertura }
file_format { :gzip }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/cobertura/coverage_gocov_xml.xml.gz'), 'application/x-gzip')
end
end
trait :coverage_with_corrupted_data do
file_type { :cobertura }
file_format { :gzip }
after(:build) do |artifact, evaluator|
artifact.file = fixture_file_upload(
Rails.root.join('spec/fixtures/cobertura/coverage_with_corrupted_data.xml.gz'), 'application/x-gzip')
end
end
trait :codequality do
file_type { :codequality }
file_format { :raw }
......
......