Java 微服务监控与日志：Prometheus + Grafana + ELK + SkyWalking + Loki 全链路可观测性实战

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

<!-- pom.xml 关键依赖 -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<dependency>
    <groupId>io.micrometer</groupId>
    <artifactId>micrometer-tracing-bridge-otel</artifactId>
</dependency>
<dependency>
    <groupId>io.opentelemetry</groupId>
    <artifactId>opentelemetry-exporter-otlp</artifactId>
</dependency>

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# application.yml
spring:
  application:
    name: order-service  # 自动成为 OTel 的 service.name

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus
  tracing:
    sampling:
      probability: 1.0  # 采样率（生产建议 0.1-0.3）
  otlp:
    tracing:
      endpoint: http://otel-collector:4318/v1/traces
      timeout: 10s

# 日志关联 TraceID（最关键的一步！）
logging:
  pattern:
    level: "%5p [${spring.application.name},%X{traceId:-},%X{spanId:-}]"

1

http_server_requests_seconds_count{method="POST", uri="/api/order", status="200"} 12345 @1717900000

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

# 当前 CPU 使用率
100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 5 分钟平均 QPS
sum(rate(http_server_requests_seconds_count[5m]))

# 错误率
sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
  /
sum(rate(http_server_requests_seconds_count[5m]))

# P99 响应时间（histogram_quantile）
histogram_quantile(0.99,
  sum by (le) (rate(http_server_requests_seconds_bucket[5m]))
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 按服务汇总 QPS
sum by (service) (rate(http_server_requests_seconds_count[5m]))

# 排序取 Top 10
topk(10, sum by (uri) (rate(http_server_requests_seconds_count[5m])))

# 增长率（同比上周）
(rate(http_server_requests_seconds_count[5m]) -
 rate(http_server_requests_seconds_count[5m] offset 1w))
 /
rate(http_server_requests_seconds_count[5m] offset 1w)

1
2
3
4

# JVM 堆内存使用率
sum(jvm_memory_used_bytes{area="heap"}) 
  / 
sum(jvm_memory_max_bytes{area="heap"})

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# prometheus.yml
scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # 只采集有特定 annotation 的 pod
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      # 重命名 label
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: service
      # 删除高基数 label
      - source_labels: [__meta_kubernetes_pod_container_name]
        action: drop

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 远程写入到 Thanos / Cortex / Mimir / 阿里云 SLS / 腾讯云 CLS
remote_write:
  - url: http://thanos-receive:19291/api/v1/receive
    write_relabel_configs:
      - source_labels: [__name__]
        regex: 'go_.*|process_.*'  # 丢弃内置指标
        action: drop
    queue_config:
      capacity: 10000
      max_samples_per_send: 2000
      batch_send_deadline: 5s

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38

# alert.rules.yml
groups:
  - name: example
    rules:
      # 规则 1：P99 响应时间 > 1s 持续 5 分钟
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            sum by (service) (rate(http_server_requests_seconds_bucket[5m]))
          ) > 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.service }} P99 > 1s"
          description: "P99 = {{ $value }}s"
          runbook_url: "https://wiki.example.com/runbook/HighP99Latency"

      # 规则 2：错误率 > 5%
      - alert: HighErrorRate
        expr: |
          sum by (service) (rate(http_server_requests_seconds_count{status=~"5.."}[5m]))
          /
          sum by (service) (rate(http_server_requests_seconds_count[5m]))
          > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.service }} 错误率 > 5%"

      # 规则 3：JVM FullGC 频繁
      - alert: JvmFullGcFrequent
        expr: |
          rate(jvm_gc_pause_seconds_count{cause="G1 Old Generation"}[5m]) > 0.1
        for: 5m
        labels:
          severity: warning

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

@RestController
public class OrderController {
    // 1. 注入 MeterRegistry（Spring Boot 自动配置）
    @Autowired
    private MeterRegistry meterRegistry;

    // 2. 计数器：下单总数（带 status 标签）
    private final Counter orderSuccessCounter = Counter.builder("order_create_total")
            .tag("channel", "app")  // 标签：app / web / mini
            .description("订单创建总数")
            .register(meterRegistry);

    // 3. Timer：订单创建耗时（自动算 P99）
    private final Timer orderCreateTimer = Timer.builder("order_create_duration")
            .publishPercentileHistogram()  // 关键：发布 histogram 才能算 P99
            .publishPercentiles(0.5, 0.9, 0.99)  // 客户端聚合分位数
            .register(meterRegistry);

    @PostMapping("/api/order")
    public Order createOrder(@RequestBody OrderRequest req) {
        // 用 Timer 包装业务逻辑
        return orderCreateTimer.record(() -> {
            Order order = orderService.create(req);

            // 业务计数
            orderSuccessCounter.increment();

            // 业务 Gauge：当前活跃订单数
            meterRegistry.gauge("active_orders",
                    Tags.of("channel", req.getChannel()),
                    order);

            return order;
        });
    }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# application.yml 暴露 Prometheus 端点
management:
  endpoints.web.exposure.include: health,info,metrics,prometheus
  metrics:
    tags:
      application: ${spring.application.name}
    distribution:
      percentiles-histogram:
        order_create_duration: true
      slo:
        order_create_duration: 100ms,500ms,1s  # 关键桶

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 5 分钟下单 QPS
sum(rate(order_create_total[5m]))

# 下单 P99 响应时间
histogram_quantile(0.99,
  sum by (le) (rate(order_create_duration_seconds_bucket[5m]))
)

# 渠道维度错误率
sum by (channel) (rate(order_create_total{status="failed"}[5m]))
  /
sum by (channel) (rate(order_create_total[5m]))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

# grafana provisioning 方式统一告警
apiVersion: 1
groups:
  - orgId: 1
    name: production-slo
    folder: SLO
    interval: 1m
    rules:
      - uid: api-availability
        title: API 可用性 < 99.9%
        condition: A
        data:
          - refId: A
            datasourceUid: prometheus
            relativeTimeRange:
              from: 600
              to: 0
            model:
              expr: |
                1 - (
                  sum(rate(http_requests_total{status=~"5.."}[5m]))
                  /
                  sum(rate(http_requests_total[5m]))
                ) < 0.999
          - refId: B
            datasourceUid: __expr__
            ...

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# filebeat.yml
filebeat.inputs:
  - type: container
    paths:
      - /var/log/containers/*.log
    processors:
      - add_kubernetes_metadata:
          host: ${NODE_NAME}
          matchers:
            - logs_path:
                logs_path: /var/log/containers/

output.logstash:
  hosts: ["logstash:5044"]
  ssl.enabled: true
  ssl.certificate_authorities: ["/etc/pki/tls/certs/logstash.crt"]

# 也可以直接写 ES
# output.elasticsearch:
#   hosts: ["${ES_HOST}:9200}"]
#   index: "app-logs-%{+yyyy.MM.dd}"
#   username: "${ES_USER}"
#   password: "${ES_PASS}"

logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat
  keepfiles: 7
  permissions: 0644

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

# logstash.conf
input {
  beats {
    port => 5044
  }
}

filter {
  # 1. 解析 JSON 日志
  if [container][image] =~ /order-service/ {
    json {
      source => "message"
      target => "app"
    }
    
    # 2. 时间字段转换
    date {
      match => ["[app][timestamp]", "ISO8601"]
      target => "@timestamp"
    }
    
    # 3. 提取 TraceID
    if [app][traceId] {
      mutate {
        add_field => { "trace_id" => "%{[app][traceId]}" }
      }
    }
    
    # 4. 错误日志标记
    if [app][level] == "ERROR" {
      mutate {
        add_tag => ["error", "needs_alert"]
      }
    }
    
    # 5. 脱敏（手机号、身份证）
    mutate {
      gsub => [
        "message", "1[3-9]\d{9}", "1**********",
        "message", "\d{17}[\dXx]", "******************"
      ]
    }
  }
}

output {
  elasticsearch {
    hosts => ["${ES_HOSTS}"]
    index => "app-logs-%{+YYYY.MM.dd}"
    user => "${ES_USER}"
    password => "${ES_PASS}"
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

# ILM Policy
PUT _ilm/policy/app-logs-policy
{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_age": "1d",
            "max_size": "50gb"
          },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "3d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "7d",
        "actions": {
          "freeze": {},
          "set_priority": { "priority": 0 }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": { "delete": {} }
      }
    }
  }
}

1
2
3
4
5
6

# 应用启动参数加 agent
java -javaagent:/path/to/skywalking-agent.jar \
     -Dskywalking.agent.service_name=order-service \
     -Dskywalking.collector.backend_service=skywalking-oap:11800 \
     -Dskywalking.plugin.springmvc.trace_get_parameter=true \
     -jar order-service.jar

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  template:
    spec:
      initContainers:
        - name: skywalking-agent
          image: apache/skywalking-java-agent:9.0.0
          command: ["sh", "-c", "cp -r /skywalking/agent /agent"]
          volumeMounts:
            - name: agent
              mountPath: /agent
      containers:
        - name: app
          env:
            - name: JAVA_TOOL_OPTIONS
              value: "-javaagent:/agent/skywalking-agent.jar"
            - name: SW_AGENT_NAME
              value: "order-service"
            - name: SW_AGENT_COLLECTOR_BACKEND_SERVICES
              value: "skywalking-oap:11800"
          volumeMounts:
            - name: agent
              mountPath: /agent
      volumes:
        - name: agent
          emptyDir: {}

1
2
3
4

# agent/config/agent.config
agent.sample=${SW_AGENT_SAMPLE:10000}  # 默认 10000 采 1 个（0.01%）
# 关键服务改 100% 采样
agent.sample=1000  # 0.1%

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# alarm-settings.yml
rules:
  - name: service_resp_time_rule
    expression: service_resp_time_percentile{app="order-service", percentile="50,75,90,95,99"} > 1000
    period: 10
    silence-period: 10
    message: 服务 {name} 平均响应时间超过 1s
    tags:
      level: WARNING
  - name: service_error_rate_rule
    expression: service_error_rate{app="order-service"} > 0.05
    period: 5
    message: 服务 {name} 错误率超过 5%
    tags:
      level: CRITICAL
  - name: service_instance_restart_rule
    expression: service_instance_restart{app="order-service"} > 0
    period: 1
    message: 服务 {name} 实例重启

webhooks:
  - url: https://oapi.dingtalk.com/robot/send?access_token=xxx
    secret: xxx

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

# skywalking-oap 集群化部署
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: skywalking-oap
spec:
  serviceName: skywalking-oap
  replicas: 3  # OAP 节点数
  template:
    spec:
      containers:
        - name: oap
          image: apache/skywalking-oap-server:9.0.0
          env:
            - name: SW_CLUSTER
              value: kubernetes
            - name: SW_CLUSTER_K8S_NAMESPACE
              value: monitoring
            - name: SW_CLUSTER_K8S_LABEL
              value: app=skywalking-oap
            - name: SW_STORAGE
              value: elasticsearch
            - name: SW_STORAGE_ES_CLUSTER_NODES
              value: elasticsearch:9200
            - name: JAVA_OPTS
              value: "-Xms4g -Xmx4g"
          ports:
            - containerPort: 11800  # gRPC
            - containerPort: 12800  # HTTP
            - containerPort: 1234   # 自监控

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# alarm-settings.yml
rules:
  # 规则 1：服务平均响应时间
  - name: service_resp_time_rule
    expression: service_resp_time_percentile{app="order-service", percentile="50,75,90,95,99"} > 1000
    period: 10
    silence-period: 10
    message: 服务 {name} 平均响应时间超过 1s
    tags:
      level: WARNING
    # 关键：多个实例的告警合并成 1 条
    include-paths:
      - service

1
2
3
4
5
6
7

# agent.config 关键配置
agent.service_name=${SW_AGENT_NAME:order-service}
agent.sample=${SW_AGENT_SAMPLE:10000}  # 默认 10000 采 1
# 错误强制采样
agent.force_sample_error_trace=true
# 慢调用强制采样（> 1s）
agent.slow_trace_threshold=1000

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# promtail 配置：提取标签
scrape_configs:
  - job_name: kubernetes-pods
    pipeline_stages:
      - docker: {}
      - cri: {}
      - match:
          selector: '{app="order-service"}'
          stages:
            - json:
                expressions:
                  level: level
                  traceId: traceId
            - labels:
                level:  # 把 level 提取成标签
            - metrics:
                log_lines_total:
                  type: Counter
                  description: total log lines
                  config:
                    match_all: true
                    action: inc

1
2
3
4
5

# 必备
{app="order-service", env="prod", region="cn-east-1", level="error"}

# 可选
{namespace="production", pod="order-7d8b-abc", container="app"}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# 1. 基础过滤
{app="order-service"} |= "ERROR"

# 2. 不匹配（排除）
{app="order-service"} != "DEBUG"

# 3. 正则
{app="order-service"} |~ "timeout|exception"

# 4. 多条件
{app="order-service", env="prod"} |~ "timeout" |~ "order_id"

1
2
3
4
5
6
7

# 错误日志数量（按 5 分钟聚合）
sum(rate({app="order-service"} |= "ERROR" [5m]))

# 按错误类型分组
sum by (error_type) (
  rate({app="order-service"} |= "ERROR" | regexp "error_type=(?P<error_type>[\\w]+)" [5m])
)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

# 创建 OTel Collector
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: otel-collector
spec:
  mode: deployment  # 或 daemonset
  config: |
    receivers:
      otlp:
        protocols:
          grpc: {endpoint: 0.0.0.0:4317}
          http: {endpoint: 0.0.0.0:4318}
      prometheus:
        config:
          scrape_configs:
            - job_name: 'kubernetes-pods'
              kubernetes_sd_configs:
                - role: pod
    processors:
      batch: {}
      memory_limiter:
        check_interval: 1s
        limit_percentage: 80
        spike_limit_percentage: 20
      resource:
        attributes:
          - key: deployment.environment
            from_attribute: k8s.namespace.labels
            action: insert
    exporters:
      prometheusremotewrite:
        endpoint: http://thanos-receive:19291/api/v1/receive
      otlp/jaeger:
        endpoint: http://skywalking-oap:11800
        tls: {insecure: true}
    service:
      pipelines:
        metrics:
          receivers: [otlp, prometheus]
          processors: [memory_limiter, batch]
          exporters: [prometheusremotewrite]
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [otlp/jaeger]
---
# 创建 Instrumentation（CRD），自动给 Pod 注入 Java Agent
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: java-instrumentation
spec:
  exporter:
    endpoint: http://otel-collector:4318
  propagators:
    - tracecontext
    - baggage
  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:1.32.0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# 在 Deployment 上引用
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-java: "true"
    spec:
      containers:
        - name: app
          image: order-service:1.0.0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 1. 添加 Helm 仓库
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# 2. 一键部署（生产推荐自定义 values）
helm install kube-prometheus prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=100Gi \
  --set grafana.adminPassword=yourStrongPassword \
  --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.resources.requests.storage=10Gi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 给应用配 ServiceMonitor，Prometheus 自动抓取
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: order-service
  labels:
    release: kube-prometheus  # 必须与 Helm release 名一致
spec:
  selector:
    matchLabels:
      app: order-service
  endpoints:
    - port: http  # Service 的端口名
      path: /actuator/prometheus
      interval: 30s
      scrapeTimeout: 10s

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: order-service-alerts
  labels:
    release: kube-prometheus
spec:
  groups:
    - name: order-service
      interval: 30s
      rules:
        - alert: OrderServiceHighErrorRate
          expr: |
            sum(rate(http_server_requests_seconds_count{service="order-service", status=~"5.."}[5m]))
            /
            sum(rate(http_server_requests_seconds_count{service="order-service"}[5m]))
            > 0.05
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "订单服务错误率 > 5%"
            runbook_url: "https://wiki.example.com/runbook/OrderServiceHighErrorRate"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'default-receiver'
  group_by: ['alertname', 'service', 'cluster']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  
  routes:
    # P0 紧急：电话 + 短信 + 钉钉
    - match_re:
        severity: critical
      receiver: 'p0-oncall'
      group_wait: 10s
      repeat_interval: 5m
      continue: true
      routes:
        - match_re:
            severity: critical
          receiver: 'p0-phone'
          # 不 group_wait，立即电话
    
    # P1 严重：钉钉 + 升级
    - match_re:
        severity: warning
      receiver: 'p1-oncall'
      group_wait: 1m
      repeat_interval: 30m
    
    # P2 警告：钉钉群消息
    - match_re:
        severity: info
      receiver: 'p2-oncall'
      group_wait: 5m
      repeat_interval: 4h

receivers:
  - name: 'p0-oncall'
    webhook_configs:
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'
        send_resolved: true
        # 钉钉 @oncall 工程师
    wechat_configs:
      - corp_id: 'xxx'
        agent_id: 'xxx'
        api_secret: 'xxx'
        to_user: 'oncall-team'
  
  - name: 'p0-phone'
    pagerduty_configs:
      - service_key: 'xxx'
        description: '{{ .CommonAnnotations.summary }}'
    # 阿里云语音通知 / 腾讯云电话告警
    webhook_configs:
      - url: 'http://voice-alert:8080/alert?phone=oncall'
  
  - name: 'p1-oncall'
    webhook_configs:
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'
  
  - name: 'p2-oncall'
    webhook_configs:
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'

# 抑制规则：避免告警风暴
inhibit_rules:
  - source_match_re:
      severity: 'critical'
    target_match_re:
      severity: 'warning'
    equal: ['alertname', 'service']
    # critical 触发时，抑制同服务 warning

1
2
3

# Alertmanager 按 alertname + service 分组
group_by: ['alertname', 'service', 'cluster']
group_wait: 30s  # 30 秒内的告警合并成一条

1
2
3
4
5
6
7

# critical 触发时抑制同服务的 warning
inhibit_rules:
  - source_match_re:
      severity: 'critical'
    target_match_re:
      severity: 'warning'
    equal: ['alertname', 'service']

1
2
3
4
5

# 升级期间静默告警
amtool silence add --alertmanager=http://localhost:9093 \
  --start="2026-06-09 22:00" --end="2026-06-10 02:00" \
  --matcher="service=order-service" \
  --comment="订单服务升级维护"

1
2

# Alertmanager 5 分钟内同 alertname + instance 只发一次
repeat_interval: 5m

1
2
3
4
5
6
7
8

# 工作时间 / 非工作时间不同通知策略
- matchers:
    - severity = "critical"
  receiver: 'business-hours-pager'
- matchers:
    - severity = "critical"
    - environment = "prod"
  receiver: 'after-hours-pager'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 短期快燃烧（5 分钟）—— 立即响应
sum(rate(http_requests_total{status=~"5.."}[5m]))
  / sum(rate(http_requests_total[5m]))
  > (14.4 * 0.001)  # 14.4x 燃烧率

# 中期快燃烧（30 分钟）—— 30min 内响应
... [30m] > (6 * 0.001)

# 长期慢燃烧（6 小时）—— 6h 内响应
... [6h] > (1 * 0.001)

# 超长期慢燃烧（3 天）—— 3d 内响应
... [3d] > (1 * 0.001)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21

# 告警：HighP99Latency（订单服务 P99 > 1s）

## 1. 确认问题
- 打开 [订单服务大盘](https://grafana.example.com/d/order-overview)
- 查看 P99 曲线，判断趋势
- 打开 [SkyWalking 慢调用面板](https://skywalking.example.com/dashboard)

## 2. 常见原因（按概率排序）
- **下游服务慢**（占 60%）：查看 SkyWalking 慢 span
- **JVM 频繁 FullGC**（占 20%）：看 jvm_gc_pause_seconds
- **数据库慢查询**（占 15%）：看 MySQL 慢日志
- **线程池耗尽**（占 5%）：看 jvm_threads_states

## 3. 应急操作
- 限流降级：执行 `sentinel-cli flowRule add order-service 5000`
- 切只读：联系 DBA 切数据库只读
- 扩容：HPA 自动扩容或手动 `kubectl scale deploy order-service --replicas=20`

## 4. 复盘要求
- 24h 内提交事故复盘
- P0/P1 必须有 RCA（Root Cause Analysis）