Метрики: Prometheus + Grafana, .NET Metrics API?

# HELP http_server_request_duration_seconds Duration of HTTP server requests.
# TYPE http_server_request_duration_seconds histogram
http_server_request_duration_seconds_bucket{http_route="/orders/{id}",http_response_status_code="200",le="0.05"} 8412
http_server_request_duration_seconds_bucket{http_route="/orders/{id}",http_response_status_code="200",le="0.1"} 9120
http_server_request_duration_seconds_bucket{http_route="/orders/{id}",http_response_status_code="200",le="+Inf"} 9301
http_server_request_duration_seconds_sum{http_route="/orders/{id}",http_response_status_code="200"} 412.7
http_server_request_duration_seconds_count{http_route="/orders/{id}",http_response_status_code="200"} 9301
# TYPE shop_orders_placed_total counter
shop_orders_placed_total{payment_method="card"} 15230

public sealed class OrderMetrics
{
    private readonly Counter<long> _placed;
    private readonly Histogram<double> _checkoutDuration;
    private readonly UpDownCounter<int> _activeCheckouts;

    public OrderMetrics(IMeterFactory meterFactory)                     // DI-friendly (.NET 8)
    {
        var meter = meterFactory.Create("Shop.Orders");
        _placed = meter.CreateCounter<long>("shop.orders.placed", unit: "{order}", description: "Оформленные заказы");
        _checkoutDuration = meter.CreateHistogram<double>("shop.checkout.duration", unit: "s");
        _activeCheckouts = meter.CreateUpDownCounter<int>("shop.checkout.active");
        meter.CreateObservableGauge("shop.outbox.pending", () => OutboxStats.Pending);   // значение читается при сборе
    }

    public void OrderPlaced(string paymentMethod) =>
        _placed.Add(1, new KeyValuePair<string, object?>("payment.method", paymentMethod));

    public IDisposable TrackCheckout()
    {
        _activeCheckouts.Add(1);
        var start = Stopwatch.GetTimestamp();
        return new Scope(() =>
        {
            _activeCheckouts.Add(-1);
            _checkoutDuration.Record(Stopwatch.GetElapsedTime(start).TotalSeconds);
        });
    }
    private sealed class Scope(Action onDispose) : IDisposable { public void Dispose() => onDispose(); }
}

builder.Services.AddOpenTelemetry()
    .WithMetrics(m => m
        .AddAspNetCoreInstrumentation()          // http.server.request.duration, active requests
        .AddHttpClientInstrumentation()          // http.client.request.duration
        .AddRuntimeInstrumentation()             // GC, потоки, исключения, память
        .AddMeter("Shop.Orders")
        .AddPrometheusExporter());               // эндпоинт /metrics (пакет OpenTelemetry.Exporter.Prometheus.AspNetCore)

app.MapPrometheusScrapingEndpoint();             // лучше на отдельном внутреннем порту

# RPS по маршрутам
sum by (http_route) (rate(http_server_request_duration_seconds_count[5m]))

# Доля ошибок 5xx
sum(rate(http_server_request_duration_seconds_count{http_response_status_code=~"5.."}[5m]))
  / sum(rate(http_server_request_duration_seconds_count[5m]))

# p99 латентности по маршруту (из гистограммы)
histogram_quantile(0.99, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket[5m])))

# Заказы в минуту по способу оплаты
sum by (payment_method) (rate(shop_orders_placed_total[1m])) * 60

# Время в GC
rate(process_runtime_dotnet_gc_pause_time_seconds_total[5m])

groups:
  - name: orders-slo
    rules:
      - alert: OrdersHighErrorRate
        expr: |
          (sum(rate(http_server_request_duration_seconds_count{service="orders-api",http_response_status_code=~"5.."}[5m]))
            / sum(rate(http_server_request_duration_seconds_count{service="orders-api"}[5m]))) > 0.02
        for: 10m
        labels: { severity: page }
        annotations:
          summary: "orders-api: более 2% ошибок 5xx в течение 10 минут"
          runbook: "https://wiki.example.uz/runbooks/orders-api-errors"