From f19cbe349ac0f3c57d979f08b264ffd825644183 Mon Sep 17 00:00:00 2001 From: dakotadux Date: Mon, 3 Nov 2025 10:04:10 -0700 Subject: [PATCH] Ship CI to Observability - Part 1 This is the first part of the series of commits to ship CI to Observability. This commit initiates the export of CI data to the GitLab o11y service. A user can enable the export of CI data to the GitLab o11y service by setting the `GITLAB_OBSERVABILITY_EXPORT` CI variable in their pipeline. ```yaml variables: GITLAB_OBSERVABILITY_EXPORT: "traces,metrics,logs" ``` The variable accepts a comma-separated list of data types to export: - `traces` - Export pipeline and job data as OpenTelemetry traces - `metrics` - Export duration and status metrics - `logs` - Export pipeline events as structured logs A user can export any combination: - `"traces"` - Only traces - `"traces,metrics"` - Traces and metrics - `"traces,metrics,logs"` - Everything (recommended) The export is triggered by the `Ci::Pipelines::HookService` when a pipeline is created. The `Ci::Observability::ExportService` is responsible for building the data and sending it to the GitLab o11y service. The `Gitlab::Observability::OtelExporter` is responsible for sending the data to the GitLab o11y service. --- .../observability/group_o11y_setting.rb | 12 + .../ci/observability/export_service.rb | 122 +++++++ app/services/ci/pipelines/hook_service.rb | 16 + app/workers/all_queues.yml | 10 + app/workers/ci/observability/export_worker.rb | 30 ++ .../observability/group_o11y_setting_spec.rb | 53 +++ .../ci/observability/export_service_spec.rb | 321 ++++++++++++++++++ .../ci/pipelines/hook_service_spec.rb | 29 ++ .../ci/observability/export_worker_spec.rb | 57 ++++ spec/workers/every_sidekiq_worker_spec.rb | 1 + 10 files changed, 651 insertions(+) create mode 100644 app/services/ci/observability/export_service.rb create mode 100644 app/workers/ci/observability/export_worker.rb create mode 100644 spec/services/ci/observability/export_service_spec.rb create mode 100644 spec/workers/ci/observability/export_worker_spec.rb diff --git a/app/models/observability/group_o11y_setting.rb b/app/models/observability/group_o11y_setting.rb index ef0050a5ee0171..85a8c6b6ea2dc0 100644 --- a/app/models/observability/group_o11y_setting.rb +++ b/app/models/observability/group_o11y_setting.rb @@ -38,6 +38,18 @@ def self.human_attribute_name(attribute, *options) HUMANIZED_ATTRIBUTES[attribute.to_sym] || super end + def self.observability_settings_for(resource) + return unless resource + + group = resource.is_a?(Project) ? resource.group : resource + return unless group.is_a?(Group) + + group.self_and_ancestors(hierarchy_order: :asc) + .lazy + .filter_map(&:observability_group_o11y_setting) + .first + end + def o11y_service_name @o11y_service_name || name_from_url || name_from_group end diff --git a/app/services/ci/observability/export_service.rb b/app/services/ci/observability/export_service.rb new file mode 100644 index 00000000000000..a347bab61fdde6 --- /dev/null +++ b/app/services/ci/observability/export_service.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Ci + module Observability + class ExportService + include Gitlab::Utils::StrongMemoize + + OBSERVABILITY_VARIABLE = 'GITLAB_OBSERVABILITY_EXPORT' + VALID_VARIABLE_VALUES = %w[traces metrics logs].freeze + + def initialize(pipeline) + @pipeline = pipeline + end + + def execute + return unless should_export? && observability_available? + + export_data + rescue StandardError => e + Gitlab::AppLogger.error( + message: "GitLab Observability export failed", + pipeline_id: pipeline.id, + project_id: pipeline.project_id, + error_class: e.class.name, + error_message: e.message + ) + end + + private + + attr_reader :pipeline + + def should_export? + export_types.present? + end + + def observability_available? + observability_settings.present? + end + + def observability_settings + ::Observability::GroupO11ySetting.observability_settings_for(pipeline.project) + end + strong_memoize_attr :observability_settings + + def export_types + build = pipeline.builds.first + return [] unless build + + variables = pipeline.variables_builder.scoped_variables( + build, + environment: nil, + dependencies: false + ) + + export_variables = variables.find { |var| var.key == OBSERVABILITY_VARIABLE } + return [] unless export_variables.present? + + export_variables.value.to_s.downcase.split(',').map(&:strip) & VALID_VARIABLE_VALUES + end + strong_memoize_attr :export_types + + def export_data + pipeline_data = Gitlab::DataBuilder::Pipeline.build(pipeline) + + export_types.each do |export_type| + case export_type + when 'traces' + export_traces(pipeline_data) + when 'metrics' + export_metrics(pipeline_data) + when 'logs' + export_logs(pipeline_data) + end + end + end + + def export_traces(pipeline_data) + traces_data = Gitlab::Observability::PipelineToTraces.new(integration, pipeline_data).convert + exporter.export_traces(traces_data) if traces_data.present? + end + + def export_metrics(pipeline_data) + metrics_data = Gitlab::Observability::PipelineToMetrics.new(integration, pipeline_data).convert + exporter.export_metrics(metrics_data) if metrics_data.present? + end + + def export_logs(pipeline_data) + logs_data = Gitlab::Observability::PipelineToLogs.new(integration, pipeline_data).convert + exporter.export_logs(logs_data) if logs_data.present? + end + + def integration + Struct.new( + :otel_endpoint_url, + :otel_headers, + :service_name, + :environment + ).new( + otel_endpoint_url, + otel_headers, + 'gitlab-ci', + Rails.env + ) + end + strong_memoize_attr :integration + + def exporter + Gitlab::Observability::OtelExporter.new(integration) + end + strong_memoize_attr :exporter + + def otel_endpoint_url + observability_settings.otel_http_endpoint + end + + def otel_headers + {} + end + end + end +end diff --git a/app/services/ci/pipelines/hook_service.rb b/app/services/ci/pipelines/hook_service.rb index 629ed7e1ebd085..469a1ef257a282 100644 --- a/app/services/ci/pipelines/hook_service.rb +++ b/app/services/ci/pipelines/hook_service.rb @@ -14,6 +14,12 @@ def initialize(pipeline) def execute project.execute_hooks(hook_data, HOOK_NAME) if project.has_active_hooks?(HOOK_NAME) project.execute_integrations(hook_data, HOOK_NAME) if project.has_active_integrations?(HOOK_NAME) + + return unless has_observability_settings? + + pipeline.run_after_commit_or_now do + Ci::Observability::ExportWorker.perform_async(id) + end end private @@ -29,6 +35,16 @@ def hook_data Gitlab::DataBuilder::Pipeline.build(pipeline) end end + + def has_observability_settings? + observability_settings.present? + end + + def observability_settings + strong_memoize(:observability_settings) do + ::Observability::GroupO11ySetting.observability_settings_for(project) + end + end end end end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index ccba1a4fc4b0f7..e6925916dcbe9e 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -2795,6 +2795,16 @@ :idempotent: false :tags: [] :queue_namespace: :pipeline_default +- :name: pipeline_hooks:ci_observability_export + :worker_name: Ci::Observability::ExportWorker + :feature_category: :observability + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :cpu + :weight: 2 + :idempotent: false + :tags: [] + :queue_namespace: :pipeline_hooks - :name: pipeline_hooks:pipeline_hooks :worker_name: PipelineHooksWorker :feature_category: :continuous_integration diff --git a/app/workers/ci/observability/export_worker.rb b/app/workers/ci/observability/export_worker.rb new file mode 100644 index 00000000000000..239c8a66bf0a87 --- /dev/null +++ b/app/workers/ci/observability/export_worker.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Ci + module Observability + class ExportWorker # rubocop:disable Scalability/IdempotentWorker -- Export operations to external systems aren't idempotent + include ApplicationWorker + + deduplicate :until_executed + queue_namespace :pipeline_hooks + worker_resource_boundary :cpu + data_consistency :delayed + sidekiq_options retry: 3 + sidekiq_options dead: false + feature_category :observability + urgency :low + + worker_has_external_dependencies! + + defer_on_database_health_signal :gitlab_main + + def perform(pipeline_id) + pipeline = Ci::Pipeline.find_by_id(pipeline_id) + return unless pipeline + return if pipeline.user&.blocked? + + Ci::Observability::ExportService.new(pipeline).execute + end + end + end +end diff --git a/spec/models/observability/group_o11y_setting_spec.rb b/spec/models/observability/group_o11y_setting_spec.rb index 7f2a780ebb2e95..188de11bbccf0f 100644 --- a/spec/models/observability/group_o11y_setting_spec.rb +++ b/spec/models/observability/group_o11y_setting_spec.rb @@ -290,4 +290,57 @@ expect(setting.group).to be_present end end + + describe '.observability_settings_for' do + context 'when resource is nil or invalid' do + it 'returns nil for nil' do + expect(described_class.observability_settings_for(nil)).to be_nil + end + + it 'returns nil for non-Project/Group resource' do + expect(described_class.observability_settings_for(build_stubbed(:user))).to be_nil + end + + it 'returns nil for Project without group' do + expect(described_class.observability_settings_for(build_stubbed(:project, group: nil))).to be_nil + end + end + + shared_examples 'traverses parent groups' do |resource_type| + let(:root_group) { create(:group) } + let(:parent_group) { create(:group, parent: root_group) } + let(:child_group) { create(:group, parent: parent_group) } + let(:resource) { resource_type == :project ? create(:project, group: child_group) : child_group } + + it "returns setting when #{resource_type} group has setting" do + setting = create(:observability_group_o11y_setting, group: child_group) + + expect(described_class.observability_settings_for(resource)).to eq(setting) + end + + it "returns setting when parent has setting" do + setting = create(:observability_group_o11y_setting, group: parent_group) + + expect(described_class.observability_settings_for(resource)).to eq(setting) + end + + it "returns setting when root ancestor has setting" do + setting = create(:observability_group_o11y_setting, group: root_group) + + expect(described_class.observability_settings_for(resource)).to eq(setting) + end + + it "returns nil when no group in hierarchy has setting" do + expect(described_class.observability_settings_for(resource)).to be_nil + end + end + + context 'when resource is a Project' do + include_examples 'traverses parent groups', :project + end + + context 'when resource is a Group' do + include_examples 'traverses parent groups', :group + end + end end diff --git a/spec/services/ci/observability/export_service_spec.rb b/spec/services/ci/observability/export_service_spec.rb new file mode 100644 index 00000000000000..0a52a3fb0f16db --- /dev/null +++ b/spec/services/ci/observability/export_service_spec.rb @@ -0,0 +1,321 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Observability::ExportService, feature_category: :observability do + let_it_be(:project) { create(:project) } + let(:pipeline) { create(:ci_pipeline, project: project) } + let(:service) { described_class.new(pipeline) } + + shared_context 'with pipeline variables setup' do + let(:build) { create(:ci_build, pipeline: pipeline) } + let(:variables_collection) { instance_double(Gitlab::Ci::Variables::Collection) } + let(:variables_builder) { instance_double(Gitlab::Ci::Variables::Builder) } + let(:builds_relation) { instance_spy(ActiveRecord::Relation, first: build) } + + before do + allow(pipeline).to receive_messages(builds: builds_relation, variables_builder: variables_builder) + allow(variables_builder).to receive(:scoped_variables).with( + build, + environment: nil, + dependencies: false + ).and_return(variables_collection) + end + end + + shared_context 'with observability settings' do |url| + let(:observability_settings) do + instance_double(Observability::GroupO11ySetting, otel_http_endpoint: url || 'http://example.com') + end + + before do + allow(Observability::GroupO11ySetting).to receive(:observability_settings_for) + .with(project) + .and_return(observability_settings) + end + end + + def create_export_variable(value) + instance_double(Gitlab::Ci::Variables::Collection::Item, + key: described_class::OBSERVABILITY_VARIABLE, + value: value, + to_s: value) + end + + describe '#execute' do + context 'when CI variable is not set' do + before do + allow(service).to receive(:should_export?).and_return(false) + end + + it 'does not export data' do + expect(service).not_to receive(:export_data) + service.execute + end + end + + context 'when observability settings are not present' do + before do + allow(service).to receive(:should_export?).and_return(true) + allow(Observability::GroupO11ySetting).to receive(:observability_settings_for) + .with(project) + .and_return(nil) + end + + it 'does not export data' do + expect(service).not_to receive(:export_data) + service.execute + end + end + + context 'when observability settings are present and CI variable is set' do + include_context 'with observability settings' + + before do + allow(service).to receive(:should_export?).and_return(true) + allow(service).to receive(:export_data) + end + + it 'exports data' do + expect(service).to receive(:export_data) + service.execute + end + end + + context 'when an error occurs' do + include_context 'with observability settings' + + before do + allow(service).to receive(:should_export?).and_return(true) + allow(service).to receive(:export_data).and_raise(StandardError, 'Test error') + end + + it 'logs the error and does not raise' do + expect(Gitlab::AppLogger).to receive(:error).with( + hash_including( + message: "GitLab Observability export failed", + pipeline_id: pipeline.id, + project_id: pipeline.project_id, + error_class: 'StandardError', + error_message: 'Test error' + ) + ) + + expect { service.execute }.not_to raise_error + end + end + end + + describe '#should_export?' do + include_context 'with pipeline variables setup' + + context 'when CI variable has a value' do + let(:export_variable) { create_export_variable('traces,metrics') } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + end + + it 'returns true' do + expect(service.send(:should_export?)).to be_truthy + end + end + + context 'when CI variable is not set' do + before do + allow(variables_collection).to receive(:find).and_return(nil) + end + + it 'returns false' do + expect(service.send(:should_export?)).to be_falsy + end + end + + context 'when CI variable is empty' do + let(:export_variable) { create_export_variable('') } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + end + + it 'returns false' do + expect(service.send(:should_export?)).to be_falsy + end + end + end + + describe '#export_types' do + include_context 'with pipeline variables setup' + + context 'when CI variable has traces and metrics' do + let(:export_variable) { create_export_variable('traces,metrics') } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + end + + it 'returns traces and metrics' do + expect(service.send(:export_types)).to contain_exactly('traces', 'metrics') + end + end + + context 'when CI variable has invalid values' do + let(:export_variable) { create_export_variable('traces,invalid,metrics') } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + end + + it 'returns only valid values' do + expect(service.send(:export_types)).to contain_exactly('traces', 'metrics') + end + end + + context 'when CI variable is not set' do + before do + allow(variables_collection).to receive(:find).and_return(nil) + end + + it 'returns empty array' do + expect(service.send(:export_types)).to eq([]) + end + end + end + + describe '#export_data' do + include_context 'with pipeline variables setup' + include_context 'with observability settings' + + let(:pipeline_data) { { object_attributes: { id: pipeline.id }, builds: [] } } + let(:exporter) { instance_double(Gitlab::Observability::OtelExporter) } + let(:integration) { instance_double(Struct) } + + before do + allow(Gitlab::DataBuilder::Pipeline).to receive(:build).with(pipeline).and_return(pipeline_data) + allow(service).to receive_messages(integration: integration, exporter: exporter) + end + + shared_examples 'exports data type' do |export_type, converter_class, export_method| + let(:export_variable) { create_export_variable(export_type) } + let(:converter) { instance_double(converter_class) } + let(:converted_data) { { data: [] } } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + allow(converter_class).to receive(:new) + .with(integration, pipeline_data) + .and_return(converter) + allow(converter).to receive(:convert).and_return(converted_data) + allow(exporter).to receive(export_method) + end + + it "calls #{export_method} with converted data" do + expect(converter_class).to receive(:new).with(integration, pipeline_data).and_return(converter) + expect(converter).to receive(:convert).and_return(converted_data) + expect(exporter).to receive(export_method).with(converted_data) + service.send(:export_data) + end + end + + it 'builds pipeline data' do + allow(variables_collection).to receive(:find).and_return(create_export_variable('traces')) + allow(Gitlab::Observability::PipelineToTraces).to receive(:new).and_return(instance_double( + Gitlab::Observability::PipelineToTraces, convert: {})) + allow(exporter).to receive(:export_traces) + + expect(Gitlab::DataBuilder::Pipeline).to receive(:build).with(pipeline) + service.send(:export_data) + end + + it_behaves_like 'exports data type', 'traces', Gitlab::Observability::PipelineToTraces, :export_traces + it_behaves_like 'exports data type', 'metrics', Gitlab::Observability::PipelineToMetrics, :export_metrics + it_behaves_like 'exports data type', 'logs', Gitlab::Observability::PipelineToLogs, :export_logs + + context 'when multiple export types are specified' do + let(:export_variable) { create_export_variable('traces,metrics,logs') } + let(:traces_converter) { instance_double(Gitlab::Observability::PipelineToTraces) } + let(:metrics_converter) { instance_double(Gitlab::Observability::PipelineToMetrics) } + let(:logs_converter) { instance_double(Gitlab::Observability::PipelineToLogs) } + let(:traces_data) { { spans: [] } } + let(:metrics_data) { { metrics: [] } } + let(:logs_data) { { logs: [] } } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + allow(Gitlab::Observability::PipelineToTraces).to receive(:new) + .with(integration, pipeline_data).and_return(traces_converter) + allow(Gitlab::Observability::PipelineToMetrics).to receive(:new) + .with(integration, pipeline_data).and_return(metrics_converter) + allow(Gitlab::Observability::PipelineToLogs).to receive(:new) + .with(integration, pipeline_data).and_return(logs_converter) + allow(traces_converter).to receive(:convert).and_return(traces_data) + allow(metrics_converter).to receive(:convert).and_return(metrics_data) + allow(logs_converter).to receive(:convert).and_return(logs_data) + allow(exporter).to receive(:export_traces) + allow(exporter).to receive(:export_metrics) + allow(exporter).to receive(:export_logs) + end + + it 'builds pipeline data once' do + expect(Gitlab::DataBuilder::Pipeline).to receive(:build).with(pipeline).once + service.send(:export_data) + end + + it 'calls all export methods' do + expect(exporter).to receive(:export_traces).with(traces_data) + expect(exporter).to receive(:export_metrics).with(metrics_data) + expect(exporter).to receive(:export_logs).with(logs_data) + service.send(:export_data) + end + end + + context 'when converter returns empty data' do + let(:export_variable) { create_export_variable('traces') } + let(:traces_converter) { instance_double(Gitlab::Observability::PipelineToTraces) } + + before do + allow(variables_collection).to receive(:find).and_return(export_variable) + allow(Gitlab::Observability::PipelineToTraces).to receive(:new) + .with(integration, pipeline_data).and_return(traces_converter) + allow(traces_converter).to receive(:convert).and_return(nil) + end + + it 'does not call exporter when data is empty' do + expect(exporter).not_to receive(:export_traces) + service.send(:export_data) + end + end + end + + describe '#integration' do + include_context 'with observability settings', 'http://test.example.com' + + it 'returns a Struct with correct fields and values' do + integration = service.send(:integration) + + expect(integration).to be_a(Struct) + expect(integration.otel_endpoint_url).to eq('http://test.example.com') + expect(integration.otel_headers).to eq({}) + expect(integration.service_name).to eq('gitlab-ci') + expect(integration.environment).to eq(Rails.env) + end + end + + describe '#exporter' do + include_context 'with observability settings' + + it 'creates a new OtelExporter with the integration' do + integration = service.send(:integration) + expect(Gitlab::Observability::OtelExporter).to receive(:new).with(integration).and_call_original + + expect(service.send(:exporter)).to be_a(Gitlab::Observability::OtelExporter) + end + + it 'memoizes the exporter' do + exporter1 = service.send(:exporter) + exporter2 = service.send(:exporter) + + expect(exporter1).to be(exporter2) + end + end +end diff --git a/spec/services/ci/pipelines/hook_service_spec.rb b/spec/services/ci/pipelines/hook_service_spec.rb index e773ae2d2c326f..ad622350d0ee4c 100644 --- a/spec/services/ci/pipelines/hook_service_spec.rb +++ b/spec/services/ci/pipelines/hook_service_spec.rb @@ -44,4 +44,33 @@ end end end + + describe '#execute observability hooks' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, :repository, namespace: group) } + let_it_be(:pipeline, reload: true) { create(:ci_empty_pipeline, :created, project: project) } + let(:observability_setting) { instance_double(Observability::GroupO11ySetting) } + + subject(:service) { described_class.new(pipeline) } + + before do + allow(project).to receive_messages( + has_active_hooks?: false, + has_active_integrations?: false + ) + allow(observability_setting).to receive(:present?).and_return(true) + allow(Observability::GroupO11ySetting).to receive(:observability_settings_for) + .with(project) + .and_return(observability_setting) + end + + it 'calls Ci::Observability::ExportWorker via run_after_commit_or_now' do + expect(pipeline).to receive(:run_after_commit_or_now) do |&block| + pipeline.instance_eval(&block) + end + expect(Ci::Observability::ExportWorker).to receive(:perform_async).with(pipeline.id) + + service.execute + end + end end diff --git a/spec/workers/ci/observability/export_worker_spec.rb b/spec/workers/ci/observability/export_worker_spec.rb new file mode 100644 index 00000000000000..307aa20224bdbd --- /dev/null +++ b/spec/workers/ci/observability/export_worker_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Observability::ExportWorker, feature_category: :observability do + describe '#perform' do + subject(:perform_export) { described_class.new.perform(pipeline_id) } + + let_it_be(:project) { create(:project) } + + context 'when pipeline exists with unblocked user' do + let(:pipeline) { create(:ci_pipeline, project: project, user: create(:user)) } + let(:pipeline_id) { pipeline.id } + + it 'calls ExportService with the pipeline' do + expect_next_instance_of(Ci::Observability::ExportService, pipeline) do |service| + expect(service).to receive(:execute) + end + + perform_export + end + end + + context 'when pipeline exists with blocked user' do + let(:pipeline) { create(:ci_pipeline, project: project, user: create(:user, :blocked)) } + let(:pipeline_id) { pipeline.id } + + it 'does not call ExportService' do + expect(Ci::Observability::ExportService).not_to receive(:new) + + perform_export + end + end + + context 'when pipeline exists without user' do + let(:pipeline) { create(:ci_pipeline, project: project, user: nil) } + let(:pipeline_id) { pipeline.id } + + it 'calls ExportService' do + expect_next_instance_of(Ci::Observability::ExportService, pipeline) do |service| + expect(service).to receive(:execute) + end + + perform_export + end + end + + context 'when pipeline does not exist' do + let(:pipeline_id) { non_existing_record_id } + + it 'does not raise exception and does not call ExportService', :aggregate_failures do + expect { perform_export }.not_to raise_error + expect(Ci::Observability::ExportService).not_to receive(:new) + end + end + end +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index d3b0932c4df07e..1f580ed22280a3 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -168,6 +168,7 @@ 'Ci::MergeRequests::AddTodoWhenBuildFailsWorker' => 3, 'Ci::Minutes::UpdateProjectAndNamespaceUsageWorker' => 3, 'Ci::Minutes::UpdateGitlabHostedRunnerMonthlyUsageWorker' => 3, + 'Ci::Observability::ExportWorker' => 3, 'Ci::PipelineArtifacts::CoverageReportWorker' => 3, 'Ci::PipelineArtifacts::CreateQualityReportWorker' => 3, 'Ci::PipelineCleanupRefWorker' => 3, -- GitLab