...@@ -274,6 +274,9 @@ gem 'sentry-raven', '~> 2.7' ...@@ -274,6 +274,9 @@ gem 'sentry-raven', '~> 2.7'
gem 'premailer-rails', '~> 1.9.7' gem 'premailer-rails', '~> 1.9.7'
# LabKit: Tracing and Correlation
gem 'gitlab-labkit', '~> 0.1.2'
# I18n # I18n
gem 'ruby_parser', '~> 3.8', require: false gem 'ruby_parser', '~> 3.8', require: false
gem 'rails-i18n', '~> 5.1' gem 'rails-i18n', '~> 5.1'
...@@ -301,12 +304,6 @@ group :metrics do ...@@ -301,12 +304,6 @@ group :metrics do
gem 'raindrops', '~> 0.18' gem 'raindrops', '~> 0.18'
end end
group :tracing do
# OpenTracing
gem 'opentracing', '~> 0.4.3'
gem 'jaeger-client', '~> 0.10.0'
end
group :development do group :development do
gem 'foreman', '~> 0.84.0' gem 'foreman', '~> 0.84.0'
gem 'brakeman', '~> 4.2', require: false gem 'brakeman', '~> 4.2', require: false
... ...
......
...@@ -286,6 +286,12 @@ GEM ...@@ -286,6 +286,12 @@ GEM
github-markup (1.7.0) github-markup (1.7.0)
gitlab-default_value_for (3.1.1) gitlab-default_value_for (3.1.1)
activerecord (>= 3.2.0, < 6.0) activerecord (>= 3.2.0, < 6.0)
gitlab-labkit (0.1.2)
actionpack (~> 5)
activesupport (~> 5)
grpc (~> 1.15)
jaeger-client (~> 0.10)
opentracing (~> 0.4)
gitlab-markup (1.7.0) gitlab-markup (1.7.0)
gitlab-sidekiq-fetcher (0.4.0) gitlab-sidekiq-fetcher (0.4.0)
sidekiq (~> 5) sidekiq (~> 5)
...@@ -571,7 +577,7 @@ GEM ...@@ -571,7 +577,7 @@ GEM
validate_email validate_email
validate_url validate_url
webfinger (>= 1.0.1) webfinger (>= 1.0.1)
opentracing (0.4.3) opentracing (0.5.0)
optimist (3.0.0) optimist (3.0.0)
org-ruby (0.9.12) org-ruby (0.9.12)
rubypants (~> 0.2) rubypants (~> 0.2)
...@@ -1050,6 +1056,7 @@ DEPENDENCIES ...@@ -1050,6 +1056,7 @@ DEPENDENCIES
gitaly-proto (~> 1.19.0) gitaly-proto (~> 1.19.0)
github-markup (~> 1.7.0) github-markup (~> 1.7.0)
gitlab-default_value_for (~> 3.1.1) gitlab-default_value_for (~> 3.1.1)
gitlab-labkit (~> 0.1.2)
gitlab-markup (~> 1.7.0) gitlab-markup (~> 1.7.0)
gitlab-sidekiq-fetcher (~> 0.4.0) gitlab-sidekiq-fetcher (~> 0.4.0)
gitlab-styles (~> 2.5) gitlab-styles (~> 2.5)
...@@ -1076,7 +1083,6 @@ DEPENDENCIES ...@@ -1076,7 +1083,6 @@ DEPENDENCIES
httparty (~> 0.16.4) httparty (~> 0.16.4)
icalendar icalendar
influxdb (~> 0.2) influxdb (~> 0.2)
jaeger-client (~> 0.10.0)
jira-ruby (~> 1.4) jira-ruby (~> 1.4)
js_regex (~> 3.1) js_regex (~> 3.1)
json-schema (~> 2.8.0) json-schema (~> 2.8.0)
...@@ -1117,7 +1123,6 @@ DEPENDENCIES ...@@ -1117,7 +1123,6 @@ DEPENDENCIES
omniauth-twitter (~> 1.4) omniauth-twitter (~> 1.4)
omniauth-ultraauth (~> 0.0.1) omniauth-ultraauth (~> 0.0.1)
omniauth_crowd (~> 2.2.0) omniauth_crowd (~> 2.2.0)
opentracing (~> 0.4.3)
org-ruby (~> 0.9.12) org-ruby (~> 0.9.12)
peek (~> 1.0.1) peek (~> 1.0.1)
peek-gc (~> 0.0.2) peek-gc (~> 0.0.2)
... ...
......
...@@ -128,7 +128,7 @@ class ApplicationController < ActionController::Base ...@@ -128,7 +128,7 @@ class ApplicationController < ActionController::Base
payload[:ua] = request.env["HTTP_USER_AGENT"] payload[:ua] = request.env["HTTP_USER_AGENT"]
payload[:remote_ip] = request.remote_ip payload[:remote_ip] = request.remote_ip
payload[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id
logged_user = auth_user logged_user = auth_user
... ...
......
---
title: Migrate correlation and tracing code to LabKit
merge_request: 25379
author:
type: other
...@@ -35,7 +35,7 @@ unless Sidekiq.server? ...@@ -35,7 +35,7 @@ unless Sidekiq.server?
end end
payload[:response] = event.payload[:response] if event.payload[:response] payload[:response] = event.payload[:response] if event.payload[:response]
payload[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id
payload payload
end end
... ...
......
...@@ -19,7 +19,7 @@ Peek.into Peek::Views::Gitaly ...@@ -19,7 +19,7 @@ Peek.into Peek::Views::Gitaly
Peek.into Peek::Views::Rblineprof Peek.into Peek::Views::Rblineprof
Peek.into Peek::Views::Redis Peek.into Peek::Views::Redis
Peek.into Peek::Views::GC Peek.into Peek::Views::GC
Peek.into Peek::Views::Tracing if Gitlab::Tracing.tracing_url_enabled? Peek.into Peek::Views::Tracing if Labkit::Tracing.tracing_url_enabled?
# rubocop:disable Naming/ClassAndModuleCamelCase # rubocop:disable Naming/ClassAndModuleCamelCase
class PEEK_DB_CLIENT class PEEK_DB_CLIENT
... ...
......
# frozen_string_literal: true # frozen_string_literal: true
if Gitlab::Tracing.enabled? if Labkit::Tracing.enabled?
require 'opentracing'
Rails.application.configure do |config| Rails.application.configure do |config|
config.middleware.insert_after Gitlab::Middleware::CorrelationId, ::Gitlab::Tracing::RackMiddleware config.middleware.insert_after Gitlab::Middleware::CorrelationId, ::Labkit::Tracing::RackMiddleware
end end
# Instrument the Sidekiq client # Instrument the Sidekiq client
Sidekiq.configure_client do |config| Sidekiq.configure_client do |config|
config.client_middleware do |chain| config.client_middleware do |chain|
chain.add Gitlab::Tracing::Sidekiq::ClientMiddleware chain.add Labkit::Tracing::Sidekiq::ClientMiddleware
end end
end end
...@@ -18,20 +16,20 @@ if Gitlab::Tracing.enabled? ...@@ -18,20 +16,20 @@ if Gitlab::Tracing.enabled?
if Sidekiq.server? if Sidekiq.server?
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.server_middleware do |chain| config.server_middleware do |chain|
chain.add Gitlab::Tracing::Sidekiq::ServerMiddleware chain.add Labkit::Tracing::Sidekiq::ServerMiddleware
end end
end end
end end
# Instrument Rails # Instrument Rails
Gitlab::Tracing::Rails::ActiveRecordSubscriber.instrument Labkit::Tracing::Rails::ActiveRecordSubscriber.instrument
Gitlab::Tracing::Rails::ActionViewSubscriber.instrument Labkit::Tracing::Rails::ActionViewSubscriber.instrument
# In multi-processed clustered architectures (puma, unicorn) don't # In multi-processed clustered architectures (puma, unicorn) don't
# start tracing until the worker processes are spawned. This works # start tracing until the worker processes are spawned. This works
# around issues when the opentracing implementation spawns threads # around issues when the opentracing implementation spawns threads
Gitlab::Cluster::LifecycleEvents.on_worker_start do Gitlab::Cluster::LifecycleEvents.on_worker_start do
tracer = Gitlab::Tracing::Factory.create_tracer(Gitlab.process_name, Gitlab::Tracing.connection_string) tracer = Labkit::Tracing::Factory.create_tracer(Gitlab.process_name, Labkit::Tracing.connection_string)
OpenTracing.global_tracer = tracer if tracer OpenTracing.global_tracer = tracer if tracer
end end
end end
...@@ -52,9 +52,9 @@ module Gitlab ...@@ -52,9 +52,9 @@ module Gitlab
end end
def self.interceptors def self.interceptors
return [] unless Gitlab::Tracing.enabled? return [] unless Labkit::Tracing.enabled?
[Gitlab::Tracing::GRPCInterceptor.instance] [Labkit::Tracing::GRPCInterceptor.instance]
end end
private_class_method :interceptors private_class_method :interceptors
...@@ -218,7 +218,7 @@ module Gitlab ...@@ -218,7 +218,7 @@ module Gitlab
feature = feature_stack && feature_stack[0] feature = feature_stack && feature_stack[0]
metadata['call_site'] = feature.to_s if feature metadata['call_site'] = feature.to_s if feature
metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage
metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id
metadata.merge!(server_feature_flags) metadata.merge!(server_feature_flags)
... ...
......
...@@ -6,7 +6,7 @@ module Gitlab ...@@ -6,7 +6,7 @@ module Gitlab
module Loggers module Loggers
class CorrelationIdLogger < ::GrapeLogging::Loggers::Base class CorrelationIdLogger < ::GrapeLogging::Loggers::Base
def parameters(_, _) def parameters(_, _)
{ Gitlab::CorrelationId::LOG_KEY => Gitlab::CorrelationId.current_id } { Labkit::Correlation::CorrelationId::LOG_KEY => Labkit::Correlation::CorrelationId.current_id }
end end
end end
end end
... ...
......
...@@ -10,7 +10,7 @@ module Gitlab ...@@ -10,7 +10,7 @@ module Gitlab
data = {} data = {}
data[:severity] = severity data[:severity] = severity
data[:time] = timestamp.utc.iso8601(3) data[:time] = timestamp.utc.iso8601(3)
data[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id
case message case message
when String when String
... ...
......
...@@ -12,7 +12,7 @@ module Gitlab ...@@ -12,7 +12,7 @@ module Gitlab
end end
def call(env) def call(env)
::Gitlab::CorrelationId.use_id(correlation_id(env)) do ::Labkit::Correlation::CorrelationId.use_id(correlation_id(env)) do
@app.call(env) @app.call(env)
end end
end end
... ...
......
...@@ -45,7 +45,7 @@ module Gitlab ...@@ -45,7 +45,7 @@ module Gitlab
context # Make sure we've set everything we know in the context context # Make sure we've set everything we know in the context
tags = { tags = {
Gitlab::CorrelationId::LOG_KEY.to_sym => Gitlab::CorrelationId.current_id Labkit::Correlation::CorrelationId::LOG_KEY.to_sym => Labkit::Correlation::CorrelationId.current_id
} }
Raven.capture_exception(exception, tags: tags, extra: extra) Raven.capture_exception(exception, tags: tags, extra: extra)
... ...
......
...@@ -4,8 +4,8 @@ module Gitlab ...@@ -4,8 +4,8 @@ module Gitlab
module SidekiqMiddleware module SidekiqMiddleware
class CorrelationInjector class CorrelationInjector
def call(worker_class, job, queue, redis_pool) def call(worker_class, job, queue, redis_pool)
job[Gitlab::CorrelationId::LOG_KEY] ||= job[Labkit::Correlation::CorrelationId::LOG_KEY] ||=
Gitlab::CorrelationId.current_or_new_id Labkit::Correlation::CorrelationId.current_or_new_id
yield yield
end end
... ...
......
...@@ -4,9 +4,9 @@ module Gitlab ...@@ -4,9 +4,9 @@ module Gitlab
module SidekiqMiddleware module SidekiqMiddleware
class CorrelationLogger class CorrelationLogger
def call(worker, job, queue) def call(worker, job, queue)
correlation_id = job[Gitlab::CorrelationId::LOG_KEY] correlation_id = job[Labkit::Correlation::CorrelationId::LOG_KEY]
Gitlab::CorrelationId.use_id(correlation_id) do Labkit::Correlation::CorrelationId.use_id(correlation_id) do
yield yield
end end
end end
... ...
......
# frozen_string_literal: true
require 'opentracing'
module Gitlab
module Tracing
module Common
def tracer
OpenTracing.global_tracer
end
# Convience method for running a block with a span
def in_tracing_span(operation_name:, tags:, child_of: nil)
scope = tracer.start_active_span(
operation_name,
child_of: child_of,
tags: tags
)
span = scope.span
# Add correlation details to the span if we have them
correlation_id = Gitlab::CorrelationId.current_id
if correlation_id
span.set_tag('correlation_id', correlation_id)
end
begin
yield span
rescue => e
log_exception_on_span(span, e)
raise e
ensure
scope.close
end
end
def postnotify_span(operation_name, start_time, end_time, tags: nil, child_of: nil, exception: nil)
span = OpenTracing.start_span(operation_name, start_time: start_time, tags: tags, child_of: child_of)
log_exception_on_span(span, exception) if exception
span.finish(end_time: end_time)
end
def log_exception_on_span(span, exception)
span.set_tag('error', true)
span.log_kv(kv_tags_for_exception(exception))
end
def kv_tags_for_exception(exception)
case exception
when Exception
{
'event': 'error',
'error.kind': exception.class.to_s,
'message': Gitlab::UrlSanitizer.sanitize(exception.message),
'stack': exception.backtrace&.join("\n")
}
else
{
'event': 'error',
'error.kind': exception.class.to_s,
'error.object': Gitlab::UrlSanitizer.sanitize(exception.to_s)
}
end
end
end
end
end
# frozen_string_literal: true
require "cgi"
module Gitlab
module Tracing
class Factory
OPENTRACING_SCHEME = "opentracing"
def self.create_tracer(service_name, connection_string)
return unless connection_string.present?
begin
opentracing_details = parse_connection_string(connection_string)
driver_name = opentracing_details[:driver_name]
case driver_name
when "jaeger"
JaegerFactory.create_tracer(service_name, opentracing_details[:options])
else
raise "Unknown driver: #{driver_name}"
end
rescue => e
# Can't create the tracer? Warn and continue sans tracer
warn "Unable to instantiate tracer: #{e}"
nil
end
end
def self.parse_connection_string(connection_string)
parsed = URI.parse(connection_string)
unless valid_uri?(parsed)
raise "Invalid tracing connection string"
end
{
driver_name: parsed.host,
options: parse_query(parsed.query)
}
end
private_class_method :parse_connection_string
def self.parse_query(query)
return {} unless query
CGI.parse(query).symbolize_keys.transform_values(&:first)
end
private_class_method :parse_query
def self.valid_uri?(uri)
return false unless uri
uri.scheme == OPENTRACING_SCHEME &&
uri.host.to_s =~ /^[a-z0-9_]+$/ &&
uri.path.empty?
end
private_class_method :valid_uri?
end
end
end
# frozen_string_literal: true
require 'opentracing'
require 'grpc'
module Gitlab
module Tracing
class GRPCInterceptor < GRPC::ClientInterceptor
include Common
include Singleton
def request_response(request:, call:, method:, metadata:)
wrap_with_tracing(method, 'unary', metadata) do
yield
end
end
def client_streamer(requests:, call:, method:, metadata:)
wrap_with_tracing(method, 'client_stream', metadata) do
yield
end
end
def server_streamer(request:, call:, method:, metadata:)
wrap_with_tracing(method, 'server_stream', metadata) do
yield
end
end
def bidi_streamer(requests:, call:, method:, metadata:)
wrap_with_tracing(method, 'bidi_stream', metadata) do
yield
end
end
private
def wrap_with_tracing(method, grpc_type, metadata)
tags = {
'component' => 'grpc',
'span.kind' => 'client',
'grpc.method' => method,
'grpc.type' => grpc_type
}
in_tracing_span(operation_name: "grpc:#{method}", tags: tags) do |span|
OpenTracing.inject(span.context, OpenTracing::FORMAT_TEXT_MAP, metadata)
yield
end
end
end
end
end
# frozen_string_literal: true
require 'jaeger/client'
module Gitlab
module Tracing
class JaegerFactory
# When the probabilistic sampler is used, by default 0.1% of requests will be traced
DEFAULT_PROBABILISTIC_RATE = 0.001
# The default port for the Jaeger agent UDP listener
DEFAULT_UDP_PORT = 6831
# Reduce this from default of 10 seconds as the Ruby jaeger
# client doesn't have overflow control, leading to very large
# messages which fail to send over UDP (max packet = 64k)
# Flush more often, with smaller packets
FLUSH_INTERVAL = 5
def self.create_tracer(service_name, options)
kwargs = {
service_name: service_name,
sampler: get_sampler(options[:sampler], options[:sampler_param]),
reporter: get_reporter(service_name, options[:http_endpoint], options[:udp_endpoint])
}.compact
extra_params = options.except(:sampler, :sampler_param, :http_endpoint, :udp_endpoint, :strict_parsing, :debug) # rubocop: disable CodeReuse/ActiveRecord
if extra_params.present?
message = "jaeger tracer: invalid option: #{extra_params.keys.join(", ")}"
if options[:strict_parsing]
raise message
else
warn message
end
end
Jaeger::Client.build(kwargs)
end
def self.get_sampler(sampler_type, sampler_param)
case sampler_type
when "probabilistic"
sampler_rate = sampler_param ? sampler_param.to_f : DEFAULT_PROBABILISTIC_RATE
Jaeger::Samplers::Probabilistic.new(rate: sampler_rate)
when "const"
const_value = sampler_param == "1"
Jaeger::Samplers::Const.new(const_value)
else
nil
end
end
private_class_method :get_sampler
def self.get_reporter(service_name, http_endpoint, udp_endpoint)
encoder = Jaeger::Encoders::ThriftEncoder.new(service_name: service_name)
if http_endpoint.present?
sender = get_http_sender(encoder, http_endpoint)
elsif udp_endpoint.present?
sender = get_udp_sender(encoder, udp_endpoint)
else
return
end
Jaeger::Reporters::RemoteReporter.new(
sender: sender,
flush_interval: FLUSH_INTERVAL
)
end
private_class_method :get_reporter
def self.get_http_sender(encoder, address)
Jaeger::HttpSender.new(
url: address,
encoder: encoder,
logger: Logger.new(STDOUT)
)
end
private_class_method :get_http_sender
def self.get_udp_sender(encoder, address)
pair = address.split(":", 2)
host = pair[0]
port = pair[1] ? pair[1].to_i : DEFAULT_UDP_PORT
Jaeger::UdpSender.new(
host: host,
port: port,
encoder: encoder,
logger: Logger.new(STDOUT)
)
end
private_class_method :get_udp_sender
end
end
end
# frozen_string_literal: true
require 'opentracing'
module Gitlab
module Tracing
class RackMiddleware
include Common
REQUEST_METHOD = 'REQUEST_METHOD'
def initialize(app)
@app = app
end
def call(env)
method = env[REQUEST_METHOD]
context = tracer.extract(OpenTracing::FORMAT_RACK, env)
tags = {
'component' => 'rack',
'span.kind' => 'server',
'http.method' => method,
'http.url' => self.class.build_sanitized_url_from_env(env)
}
in_tracing_span(operation_name: "http:#{method}", child_of: context, tags: tags) do |span|
@app.call(env).tap do |status_code, _headers, _body|
span.set_tag('http.status_code', status_code)
end
end
end
# Generate a sanitized (safe) request URL from the rack environment
def self.build_sanitized_url_from_env(env)
request = ActionDispatch::Request.new(env)
original_url = request.original_url
uri = URI.parse(original_url)
uri.query = request.filtered_parameters.to_query if uri.query.present?
uri.to_s
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Tracing
module Rails
class ActionViewSubscriber
include RailsCommon
COMPONENT_TAG = 'ActionView'
RENDER_TEMPLATE_NOTIFICATION_TOPIC = 'render_template.action_view'
RENDER_COLLECTION_NOTIFICATION_TOPIC = 'render_collection.action_view'
RENDER_PARTIAL_NOTIFICATION_TOPIC = 'render_partial.action_view'
# Instruments Rails ActionView events for opentracing.
# Returns a lambda, which, when called will unsubscribe from the notifications
def self.instrument
subscriber = new
subscriptions = [
ActiveSupport::Notifications.subscribe(RENDER_TEMPLATE_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
subscriber.notify_render_template(start, finish, payload)
end,
ActiveSupport::Notifications.subscribe(RENDER_COLLECTION_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
subscriber.notify_render_collection(start, finish, payload)
end,
ActiveSupport::Notifications.subscribe(RENDER_PARTIAL_NOTIFICATION_TOPIC) do |_, start, finish, _, payload|
subscriber.notify_render_partial(start, finish, payload)
end
]
create_unsubscriber subscriptions
end
# For more information on the payloads: https://guides.rubyonrails.org/active_support_instrumentation.html
def notify_render_template(start, finish, payload)
generate_span_for_notification("render_template", start, finish, payload, tags_for_render_template(payload))
end
def notify_render_collection(start, finish, payload)
generate_span_for_notification("render_collection", start, finish, payload, tags_for_render_collection(payload))
end
def notify_render_partial(start, finish, payload)
generate_span_for_notification("render_partial", start, finish, payload, tags_for_render_partial(payload))
end
private
def tags_for_render_template(payload)
{
'component' => COMPONENT_TAG,
'template.id' => payload[:identifier],
'template.layout' => payload[:layout]
}
end
def tags_for_render_collection(payload)
{
'component' => COMPONENT_TAG,
'template.id' => payload[:identifier],
'template.count' => payload[:count] || 0,
'template.cache.hits' => payload[:cache_hits] || 0
}
end
def tags_for_render_partial(payload)
{
'component' => COMPONENT_TAG,
'template.id' => payload[:identifier]
}
end
end
end
end
end