Monitoring & Observability

curl http://localhost:8000/health

{
  "status": "ok",
  "time": "2026-02-02T10:30:00Z"
}

livenessProbe:
  httpGet:
    path: /health
    port: 8000
  periodSeconds: 30
  failureThreshold: 3

readinessProbe:
  httpGet:
    path: /health
    port: 8000
  periodSeconds: 10
  failureThreshold: 2

scrape_configs:
  - job_name: 'cadence'
    static_configs:
      - targets: ['localhost:8000']
    metrics_path: '/metrics'
    scrape_interval: 15s

# Throughput (jobs/min)
rate(cadence_jobs_total[5m])

# P95 latency
histogram_quantile(0.95, cadence_jobs_duration_seconds_bucket)

# Cache hit rate
rate(cadence_cache_hits_total[5m]) / (rate(cadence_cache_hits_total[5m]) + rate(cadence_cache_misses_total[5m]))

{
  "timestamp": "2026-02-02T10:30:00Z",
  "level": "info",
  "component": "webhook",
  "message": "Job completed",
  "duration_ms": 1250
}

logging:
  level: "info"
  format: "json"
  output: "stdout"

filebeat.inputs:
  - type: filestream
    paths:
      - /var/log/cadence/*.log

output.elasticsearch:
  hosts: ["elasticsearch:9200"]
  index: "cadence-%{+yyyy.MM.dd}"

{
  "logs": {
    "logs_collected": {
      "files": {
        "collect_list": [{
          "file_path": "/var/log/cadence/*.log",
          "log_group_name": "/aws/cadence/webhook"
        }]
      }
    }
  }
}

groups:
  - name: cadence-alerts
    rules:
      - alert: CadenceDown
        expr: up{job="cadence"} == 0
        for: 1m

      - alert: HighErrorRate
        expr: rate(cadence_analysis_errors_total[5m]) > 0.1
        for: 5m

      - alert: QueueBackup
        expr: cadence_queue_depth > 100
        for: 10m

# Check metrics
curl http://localhost:8000/metrics | grep memory

# Reduce workers
CADENCE_WEBHOOK_MAX_WORKERS=4

# Check queue
curl http://localhost:8000/metrics | grep queue_depth

# Increase workers
CADENCE_WEBHOOK_MAX_WORKERS=16

# Check logs
journalctl -u cadence -p err -n 50

# Get metrics
curl http://localhost:8000/metrics | grep error