From 26b02ac94dfa2f457fe14b206c55e5f911a833f2 Mon Sep 17 00:00:00 2001 From: Macocian Alexandru Victor Date: Fri, 23 Jan 2026 16:02:04 +0100 Subject: [PATCH] Improve telemetry (#1337) (Closes #1335) * Improve telemetry (Closes #1335) * Fix CD pipeline --- .github/workflows/cd.yaml | 3 +- .../Models/Plugins/PluginConfigurationBase.cs | 12 +--- .../Utils/MetricsHttpMessageHandler.cs | 57 ------------------- Daybreak/Configuration/SecretKeys.cs | 3 +- Daybreak/Services/Telemetry/TelemetryHost.cs | 55 +++--------------- 5 files changed, 13 insertions(+), 117 deletions(-) delete mode 100644 Daybreak.Shared/Utils/MetricsHttpMessageHandler.cs diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 0fc0460e..6e8996dc 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -69,7 +69,8 @@ jobs: run: | dotnet user-secrets --project Daybreak\Daybreak.csproj set AadApplicationId "${{ secrets.AadApplicationId }}" dotnet user-secrets --project Daybreak\Daybreak.csproj set AadTenantId "${{ secrets.AadTenantId }}" - dotnet user-secrets --project Daybreak\Daybreak.csproj set ApmApiKey "${{ secrets.ApmApiKey }}" + dotnet user-secrets --project Daybreak\Daybreak.csproj set ApmServiceAccount "${{ secrets.ApmServiceAccount }}" + dotnet user-secrets --project Daybreak\Daybreak.csproj set ApmServiceKey "${{ secrets.ApmServiceKey }}" dotnet user-secrets --project Daybreak\Daybreak.csproj set ApmUri "${{ secrets.ApmUri }}" - name: Restore project diff --git a/Daybreak.Shared/Models/Plugins/PluginConfigurationBase.cs b/Daybreak.Shared/Models/Plugins/PluginConfigurationBase.cs index 5ce8dde7..b471e7b4 100644 --- a/Daybreak.Shared/Models/Plugins/PluginConfigurationBase.cs +++ b/Daybreak.Shared/Models/Plugins/PluginConfigurationBase.cs @@ -1,8 +1,5 @@ -using Daybreak.Shared.Services.ApplicationArguments; -using Daybreak.Shared.Services.FileProviders; +using Daybreak.Shared.Services.FileProviders; using Daybreak.Shared.Services.Initialization; -using Daybreak.Shared.Services.Metrics; -using Daybreak.Shared.Utils; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http.Logging; using Microsoft.Extensions.Logging; @@ -35,12 +32,7 @@ public abstract class PluginConfigurationBase public static HttpMessageHandler SetupLoggingAndMetrics(IServiceProvider serviceProvider) { var logger = serviceProvider.GetRequiredService>(); - var metricsService = serviceProvider.GetRequiredService(); - - - return new MetricsHttpMessageHandler( - metricsService, - new LoggingHttpMessageHandler(logger!) { InnerHandler = new HttpClientHandler() }); + return new LoggingHttpMessageHandler(logger) { InnerHandler = new HttpClientHandler() }; } public static void SetupDaybreakUserAgent(HttpRequestHeaders httpRequestHeaders) diff --git a/Daybreak.Shared/Utils/MetricsHttpMessageHandler.cs b/Daybreak.Shared/Utils/MetricsHttpMessageHandler.cs deleted file mode 100644 index a50475cf..00000000 --- a/Daybreak.Shared/Utils/MetricsHttpMessageHandler.cs +++ /dev/null @@ -1,57 +0,0 @@ -using Daybreak.Shared.Services.Metrics; -using System.Diagnostics; -using System.Diagnostics.Metrics; -using System.Text.RegularExpressions; - -namespace Daybreak.Shared.Utils; - -public sealed partial class MetricsHttpMessageHandler : DelegatingHandler -{ - private const string LatencyUnitsName = "ms"; - private const string LatencyDescription = "Http Request Latency"; - - private readonly string name; - private readonly Histogram latencyHistogram; - - public MetricsHttpMessageHandler(IMetricsService metricsService, HttpMessageHandler innerHandler) : base(innerHandler) - { - // Sanitize the type name to be OpenTelemetry compliant - // OpenTelemetry instrument names must match: [a-zA-Z][a-zA-Z0-9_.\-/]* - var typeName = typeof(T).Name; - // Remove generic arity suffix (e.g., `1, `2) and replace invalid characters - var sanitizedName = InvalidCharsRegex().Replace(typeName, "").ToLower(); - this.name = $"http.client.latency.{sanitizedName}"; - this.latencyHistogram = metricsService.CreateHistogram(this.name, LatencyUnitsName, LatencyDescription, Models.Metrics.AggregationTypes.P95); - } - - protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) - { - var sw = Stopwatch.StartNew(); - try - { - var response = base.Send(request, cancellationToken); - return response; - } - finally - { - this.latencyHistogram.Record(sw.ElapsedMilliseconds); - } - } - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - var sw = Stopwatch.StartNew(); - try - { - var response = await base.SendAsync(request, cancellationToken); - return response; - } - finally - { - this.latencyHistogram.Record(sw.ElapsedMilliseconds); - } - } - - [GeneratedRegex(@"[^a-zA-Z0-9_.\-/]")] - private static partial Regex InvalidCharsRegex(); -} diff --git a/Daybreak/Configuration/SecretKeys.cs b/Daybreak/Configuration/SecretKeys.cs index 0438867d..1a87327d 100644 --- a/Daybreak/Configuration/SecretKeys.cs +++ b/Daybreak/Configuration/SecretKeys.cs @@ -4,8 +4,9 @@ public class SecretKeys { public static readonly SecretKeys AadApplicationId = new() { Key = "AadApplicationId" }; public static readonly SecretKeys AadTenantId = new() { Key = "AadTenantId" }; - public static readonly SecretKeys ApmApiKey = new() { Key = "ApmApiKey" }; public static readonly SecretKeys ApmUri = new() { Key = "ApmUri" }; + public static readonly SecretKeys ApmServiceAccount = new() { Key = "ApmServiceAccount" }; + public static readonly SecretKeys ApmServiceKey = new() { Key = "ApmServiceKey" }; public string Key { get; private set; } = string.Empty; private SecretKeys() diff --git a/Daybreak/Services/Telemetry/TelemetryHost.cs b/Daybreak/Services/Telemetry/TelemetryHost.cs index 3f0b8a47..ae926297 100644 --- a/Daybreak/Services/Telemetry/TelemetryHost.cs +++ b/Daybreak/Services/Telemetry/TelemetryHost.cs @@ -14,7 +14,6 @@ using System.Core.Extensions; using System.Diagnostics; using System.Diagnostics.Tracing; using System.Extensions.Core; -using System.Xml.Linq; using TrailBlazr.Services; namespace Daybreak.Services.Telemetry; @@ -26,7 +25,8 @@ internal sealed class TelemetryHost : IDisposable, IHostedService private const string MaskedValue = "[REDACTED]"; private static readonly string ApmEndpoint = SecretManager.GetSecret(SecretKeys.ApmUri); - private static readonly string ApmApiKey = SecretManager.GetSecret(SecretKeys.ApmApiKey); + private static readonly string ApmServiceAccount = SecretManager.GetSecret(SecretKeys.ApmServiceAccount); + private static readonly string ApmServiceKey = SecretManager.GetSecret(SecretKeys.ApmServiceKey); // Headers that should be masked for security private static readonly HashSet SensitiveHeaders = new(StringComparer.OrdinalIgnoreCase) @@ -50,7 +50,6 @@ internal sealed class TelemetryHost : IDisposable, IHostedService private readonly ILoggerFactory loggerFactory; private readonly ILogger logger; - private TracerProvider? tracer; private MeterProvider? meter; private OpenTelemetryLoggerProvider? otlpLoggerProvider; private ILogger? otlpLogger; @@ -89,10 +88,8 @@ internal sealed class TelemetryHost : IDisposable, IHostedService public void Dispose() { TelemetryLogSink.Instance.LoggingHandler = default; - this.tracer?.Dispose(); this.meter?.Dispose(); this.otlpLoggerProvider?.Dispose(); - this.tracer = default; this.meter = default; this.otlpLoggerProvider = default; this.otlpLogger = default; @@ -145,9 +142,7 @@ internal sealed class TelemetryHost : IDisposable, IHostedService { var scopedLogger = this.logger.CreateScopedLogger(); TelemetryLogSink.Instance.LoggingHandler = default; - this.tracer?.Dispose(); this.meter?.Dispose(); - this.tracer = default; this.meter = default; this.otlpLoggerProvider?.Dispose(); this.otlpLoggerProvider = default; @@ -158,50 +153,14 @@ internal sealed class TelemetryHost : IDisposable, IHostedService return; } - if (string.IsNullOrWhiteSpace(ApmApiKey) || string.IsNullOrWhiteSpace(ApmEndpoint)) + if (string.IsNullOrWhiteSpace(ApmServiceKey) || string.IsNullOrWhiteSpace(ApmEndpoint) || string.IsNullOrWhiteSpace(ApmServiceAccount)) { scopedLogger.LogError("ApmApiKey or ApmEndpoint is not configured. Telemetry is disabled."); return; } var apmUri = new Uri(ApmEndpoint); - this.tracer = Sdk.CreateTracerProviderBuilder() - .SetResourceBuilder(this.resourceBuilder) - .AddSource(ServiceName) - .ConfigureResource(r => r.AddService( - serviceName: ServiceName, - serviceVersion: ProjectConfiguration.CurrentVersion.ToString())) - .AddHttpClientInstrumentation(o => - { - o.RecordException = true; - o.EnrichWithHttpRequestMessage = (activity, request) => - { - activity.DisplayName = request.RequestUri?.IdnHost ?? UnknownHost; - activity.SetTag("http.url", request.RequestUri?.ToString() ?? string.Empty); - activity.SetTag("http.method", request.Method.Method); - AddRequestHeaders(activity, request); - }; - o.EnrichWithHttpResponseMessage = (activity, response) => - { - activity.SetTag("http.status_code", (int)response.StatusCode); - activity.SetStatus(response.IsSuccessStatusCode ? ActivityStatusCode.Ok : ActivityStatusCode.Error); - AddResponseHeaders(activity, response); - }; - o.EnrichWithException = (activity, exception) => - { - activity.SetTag("exception.type", exception.GetType().FullName); - activity.SetTag("exception.message", exception.Message); - activity.SetTag("exception.stacktrace", exception.StackTrace); - }; - }) - .AddSqlClientInstrumentation() - .AddOtlpExporter(o => - { - o.Endpoint = new Uri(apmUri, "v1/traces"); - o.Headers = $"Authorization=ApiKey {ApmApiKey}"; - o.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; - }) - .Build(); + var creds = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes($"{ApmServiceAccount}:{ApmServiceKey}")); this.meter = Sdk.CreateMeterProviderBuilder() .SetResourceBuilder(this.resourceBuilder) @@ -214,9 +173,9 @@ internal sealed class TelemetryHost : IDisposable, IHostedService .AddOtlpExporter((exporterOptions, readerOptions) => { exporterOptions.Endpoint = new Uri(apmUri, "v1/metrics"); - exporterOptions.Headers = $"Authorization=ApiKey {ApmApiKey}"; + exporterOptions.Headers = $"Authorization=Basic {creds}"; exporterOptions.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; - readerOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 300_000; + readerOptions.PeriodicExportingMetricReaderOptions.ExportIntervalMilliseconds = 10_000; readerOptions.TemporalityPreference = MetricReaderTemporalityPreference.Delta; }) .Build(); @@ -231,7 +190,7 @@ internal sealed class TelemetryHost : IDisposable, IHostedService logOpts.AddOtlpExporter(exp => { exp.Endpoint = new Uri(apmUri, "v1/logs"); - exp.Headers = $"Authorization=ApiKey {ApmApiKey}"; + exp.Headers = $"Authorization=Basic {creds}"; exp.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf; });