# docker-compose.yml
version: "3.8"

networks:
  observability:
    driver: bridge

volumes:
  prometheus_data:
  loki_data:
  tempo_data:
  grafana_data:

services:
  prometheus:
    image: prom/prometheus:v2.51.0
    container_name: prometheus
    restart: unless-stopped
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--storage.tsdb.retention.time=30d"
      - "--web.enable-lifecycle"
      - "--web.enable-remote-write-receiver"
    volumes:
      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./alerts:/etc/prometheus/alerts:ro
      - prometheus_data:/prometheus
    ports:
      - "9090:9090"
    networks:
      - observability

  loki:
    image: grafana/loki:2.9.7
    container_name: loki
    restart: unless-stopped
    command: -config.file=/etc/loki/loki-config.yaml
    volumes:
      - ./config/loki-config.yaml:/etc/loki/loki-config.yaml:ro
      - loki_data:/loki
    ports:
      - "3100:3100"
    networks:
      - observability

  tempo:
    image: grafana/tempo:2.4.1
    container_name: tempo
    restart: unless-stopped
    command: -config.file=/etc/tempo/tempo.yaml
    volumes:
      - ./config/tempo.yaml:/etc/tempo/tempo.yaml:ro
      - tempo_data:/var/tempo
    ports:
      - "3200:3200"
      - "4317:4317"
      - "4318:4318"
    networks:
      - observability

  grafana:
    image: grafana/grafana:10.4.2
    container_name: grafana
    restart: unless-stopped
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
    ports:
      - "3000:3000"
    networks:
      - observability

  alertmanager:
    image: prom/alertmanager:v0.27.0
    container_name: alertmanager
    restart: unless-stopped
    volumes:
      - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
      - ./config/slack.tmpl:/etc/alertmanager/slack.tmpl:ro
    ports:
      - "9093:9093"
    networks:
      - observability

  node-exporter:
    image: prom/node-exporter:v1.7.0
    container_name: node-exporter
    restart: unless-stopped
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    ports:
      - "9100:9100"
    networks:
      - observability

  blackbox-exporter:
    image: prom/blackbox-exporter:v0.25.0
    container_name: blackbox-exporter
    restart: unless-stopped
    volumes:
      - ./config/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
    ports:
      - "9115:9115"
    networks:
      - observability

  pushgateway:
    image: prom/pushgateway:v1.7.0
    container_name: pushgateway
    restart: unless-stopped
    ports:
      - "9091:9091"
    networks:
      - observability

  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.98.0
    container_name: otel-collector
    restart: unless-stopped
    command: ["--config=/etc/otel/otel-collector.yaml"]
    volumes:
      - ./config/otel-collector.yaml:/etc/otel/otel-collector.yaml:ro
    ports:
      - "4319:4317"
      - "4320:4318"
      - "8888:8888"
    networks:
      - observability

docker compose up -d

config/
├── prometheus.yml        # Scrape configs + recording rules
├── alertmanager.yml      # Route trees + inhibition rules
├── loki-config.yaml      # Log ingestion + 30d retention
├── tempo.yaml            # Trace storage + 30d retention
├── otel-collector.yaml   # Trace and log pipeline
└── blackbox.yml          # HTTP + SSL probe modules

alerts/
├── infrastructure.yml    # CPU, memory, disk, host down
├── slo-burnrate.yml      # Multi-window burn rate alerts
└── cicd.yml              # DORA threshold alerts

grafana/
├── provisioning/         # Datasource + dashboard discovery
└── dashboards/           # 5 JSON dashboards

# config/prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - /etc/prometheus/alerts/infrastructure.yml
  - /etc/prometheus/alerts/slo-burnrate.yml
  - /etc/prometheus/alerts/cicd.yml

scrape_configs:
  - job_name: node-exporter
    scrape_interval: 15s
    static_configs:
      - targets: ["node-exporter:9100"]

  - job_name: blackbox-http
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
          - http://grafana:3000
          - http://prometheus:9090/-/healthy
          - http://loki:3100/ready
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

  - job_name: pushgateway
    honor_labels: true
    static_configs:
      - targets: ["pushgateway:9091"]

# p95 latency for successful requests
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket{status!~"5.."}[5m])) by (le, job)
)

# p95 latency for error requests (errors are often faster — fail fast)
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket{status=~"5.."}[5m])) by (le, job)
)

# Requests per second
sum(rate(http_requests_total[1m])) by (job)

# Error rate as a ratio (0 = perfect, 1 = everything failing)
sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
/
sum(rate(http_requests_total[5m])) by (job)

# Memory saturation
1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)

# CPU saturation
1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))

# Disk saturation
1 - (
  node_filesystem_avail_bytes{mountpoint="/", fstype!="tmpfs"}
  / node_filesystem_size_bytes{mountpoint="/", fstype!="tmpfs"}
)

# alerts/slo-burnrate.yml
groups:
  - name: slo.recording_rules
    interval: 30s
    rules:
      - record: slo:availability:ratio_rate5m
        expr: avg_over_time(probe_success[5m])

      - record: slo:availability:ratio_rate1h
        expr: avg_over_time(probe_success[1h])

      - record: slo:availability:ratio_rate6h
        expr: avg_over_time(probe_success[6h])

      - record: slo:availability:ratio_rate30d
        expr: avg_over_time(probe_success[30d])

      # Burn rate = how fast we're consuming error budget
      # Error budget = 1 - 0.995 = 0.005
      - record: slo:availability:burn_rate1h
        expr: (1 - slo:availability:ratio_rate1h) / 0.005

      - record: slo:availability:burn_rate6h
        expr: (1 - slo:availability:ratio_rate6h) / 0.005

Budget remaining > 50%  → Deploy freely, feature work continues
Budget remaining 25-50% → Investigate incidents, no major changes
Budget remaining < 25%  → Reliability sprint, senior review on all deploys
Budget remaining 0%     → Feature freeze until budget recovers

# .github/workflows/deploy.yml
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Record deploy start time
        id: timing
        run: echo "start_ts=$(date +%s)" >> $GITHUB_OUTPUT

      - name: Build and deploy
        run: |
          echo "Your actual build and deploy steps here"

      - name: Push DORA metrics on success
        if: success()
        run: |
          LEAD_TIME=$(( $(date +%s) - ${{ steps.timing.outputs.start_ts }} ))
          WORKFLOW="${{ github.workflow }}"

          # Deployment counter
          cat <<EOF | curl --data-binary @- "${PUSHGATEWAY_URL}/metrics/job/github_actions"
          deployment_total{status="success",workflow="${WORKFLOW}"} 1
          EOF

          # Lead time
          cat <<EOF | curl --data-binary @- "${PUSHGATEWAY_URL}/metrics/job/github_actions"
          deployment_lead_time_seconds{workflow="${WORKFLOW}"} ${LEAD_TIME}
          EOF

      - name: Push DORA metrics on failure
        if: failure()
        run: |
          WORKFLOW="${{ github.workflow }}"
          cat <<EOF | curl --data-binary @- "${PUSHGATEWAY_URL}/metrics/job/github_actions"
          deployment_total{status="failure",workflow="${WORKFLOW}"} 1
          EOF

groups:
  - name: cicd.recording_rules
    rules:
      # Deployment frequency
      - record: dora:deployment_frequency:rate24h
        expr: sum(increase(deployment_total[24h])) by (workflow)

      # Change Failure Rate = failed / total over 7 days
      - record: dora:change_failure_rate:ratio7d
        expr: |
          sum(increase(deployment_total{status="failure"}[7d])) by (workflow)
          /
          sum(increase(deployment_total[7d])) by (workflow)

      # Mean Time to Restore
      - record: dora:mttr:avg7d
        expr: avg_over_time(deployment_restore_time_seconds[7d])

# grafana/provisioning/datasources/datasources.yaml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    uid: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true

  - name: Loki
    type: loki
    uid: loki
    access: proxy
    url: http://loki:3100
    jsonData:
      # This is the key config for trace drill-down
      derivedFields:
        - name: TraceID
          matcherRegex: 'traceID=(\w+)'
          url: "${__value.raw}"
          datasourceUid: tempo
          urlDisplayLabel: "Open in Tempo"
        - name: TraceID_json
          matcherRegex: '"traceId":"(\w+)"'
          url: "${__value.raw}"
          datasourceUid: tempo
          urlDisplayLabel: "Open trace in Tempo"

  - name: Tempo
    type: tempo
    uid: tempo
    access: proxy
    url: http://tempo:3200
    jsonData:
      tracesToLogsV2:
        datasourceUid: loki
        filterByTraceID: true
        customQuery: true
        query: '{service_name="${__span.tags.service.name}"} |= "${__trace.traceId}"'

Monitoring: "Something is wrong"
Observability: "Here is exactly why, where, and when"

# alerts/infrastructure.yml
groups:
  - name: infrastructure.rules
    rules:
      # Recording rules — pre-compute SLIs
      - record: sli:node_cpu_saturation
        expr: 1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))

      - record: sli:node_memory_saturation
        expr: 1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)

      # CPU alerts
      - alert: HighCPUWarning
        expr: sli:node_cpu_saturation > 0.80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU is {{ $value | humanizePercentage }} (threshold: 80%)"
          dashboard_url: "http://YOUR_SERVER_IP:3000/d/node-exporter"
          runbook_url: "https://github.com/AirFluke/meetmind-observability/blob/main/runbooks/high-cpu.md"

      - alert: HighCPUCritical
        expr: sli:node_cpu_saturation > 0.90
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Critical CPU on {{ $labels.instance }}"
          description: "CPU is {{ $value | humanizePercentage }} for 10+ minutes"
          dashboard_url: "http://YOUR_SERVER_IP:3000/d/node-exporter"
          runbook_url: "https://github.com/AirFluke/meetmind-observability/blob/main/runbooks/high-cpu.md"

      # Host down — Blackbox probe fails for 2 minutes
      - alert: HostDown
        expr: probe_success == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Host {{ $labels.instance }} is down"
          description: "Blackbox probe failed for 2+ consecutive minutes"
          runbook_url: "https://github.com/AirFluke/meetmind-observability/blob/main/runbooks/host-down.md"

# alerts/slo-burnrate.yml
  - name: slo.alerts
    rules:
      # Fast burn — act immediately
      # 14.4x means 2% of monthly budget gone in 1 hour
      - alert: SLOAvailabilityFastBurn
        expr: slo:availability:burn_rate1h > 14.4
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Fast error budget burn — act immediately"
          description: >
            Burn rate is {{ $value | humanize }}x. At this rate,
            2% of the 30-day budget will be consumed in 1 hour.
          runbook_url: "https://github.com/AirFluke/meetmind-observability/blob/main/runbooks/slo-fast-burn.md"

      # Slow burn — investigate before it escalates
      # 5x means 5% of monthly budget gone in 6 hours
      - alert: SLOAvailabilitySlowBurn
        expr: slo:availability:burn_rate6h > 5
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Slow error budget burn — investigate soon"
          description: >
            Burn rate is {{ $value | humanize }}x over 6h.
            5% of the 30-day budget will be consumed in 6 hours.
          runbook_url: "https://github.com/AirFluke/meetmind-observability/blob/main/runbooks/slo-slow-burn.md"

# config/alertmanager.yml
route:
  receiver: slack-default
  group_by: [alertname, severity, instance]
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: slack-critical
      group_wait: 10s
      repeat_interval: 4h

inhibit_rules:
  # When host is completely down, suppress CPU/memory/latency noise
  - source_match:
      alertname: HostDown
    target_match_re:
      alertname: "HighCPU.*|HighMemory.*|HighLatency.*|DiskSpace.*"
    equal: [instance]

  # Critical suppresses warning for same alert on same host
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: [alertname, instance]

# config/slack.tmpl
{{ define "slack.title" -}}
[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}
{{- end }}

{{ define "slack.body" -}}
{{ range .Alerts }}
*Alert:* {{ .Labels.alertname }}
*Severity:* {{ .Labels.severity | toUpper }}
*Status:* {{ if eq $.Status "resolved" }}✅ RESOLVED{{ else }}🔥 FIRING{{ end }}
*Host:* {{ .Labels.instance }}
*Summary:* {{ .Annotations.summary }}

*Links:*
• <{{ .Annotations.dashboard_url }}|📊 Grafana Dashboard>
• <{{ .Annotations.runbook_url }}|📖 Runbook>

*Started:* {{ .StartsAt.Format "2006-01-02 15:04:05 UTC" }}
{{ end }}
{{- end }}

# Runbook: High CPU Usage

## What is this alert?
HighCPUWarning fires when CPU exceeds 80% for 5+ minutes.

## Likely cause
1. Traffic spike
2. Runaway process
3. Post-deployment regression

## First 3 investigation steps
1. Check running processes:

2. Correlate with traffic on Unified Observability dashboard
3. Check recent deployments in GitHub Actions

## Resolution
- Runaway process: kill -9 <PID>
- Traffic spike: scale horizontally
- Deployment regression: roll back

## Roll back when?
If CPU spike started within 30 minutes of a deployment
and correlates with increased error rate.

## Escalation
Senior engineer if unresolved after 20 minutes.

sudo tc qdisc add dev ens5 root netem delay 600ms

# Remove latency
sudo tc qdisc del dev ens5 root

stress-ng --cpu 0 --cpu-method matrixprod --timeout 600s &

pkill stress-ng