当前位置：首页 > news >正文

Kubernetes可观测性体系深度解析：构建全面的监控与追踪系统

news 2026/5/27 0:52:25

Kubernetes可观测性体系深度解析：构建全面的监控与追踪系统

一、可观测性概述

可观测性是指通过系统的外部输出推断其内部状态的能力。在Kubernetes中，可观测性体系包括指标监控、日志收集和分布式追踪三个核心维度。

1.1 可观测性三大支柱

支柱	说明	工具
指标(Metrics)	数值型数据，用于监控系统状态	Prometheus
日志(Logging)	事件记录，用于问题排查	Loki、ELK
追踪(Tracing)	请求链路追踪，用于性能分析	Jaeger

1.2 可观测性架构

┌─────────────────────────┐ │ 数据采集层 │ │ (Exporters/Agents) │ └───────────┬─────────────┘ │ ┌───────────────────────┼───────────────────────┐ │ │ │ ▼ ▼ ▼ ┌───────────────┐ ┌───────────────┐ ┌───────────────┐ │ Prometheus │ │ Loki │ │ Jaeger │ │ (指标存储) │ │ (日志存储) │ │ (追踪存储) │ └───────┬───────┘ └───────┬───────┘ └───────┬───────┘ │ │ │ └───────────────────────┼───────────────────────┘ │ ┌───────────▼─────────────┐ │ Grafana │ │ (可视化展示层) │ └───────────────────────┘

二、指标监控配置

2.1 Prometheus部署

apiVersion: monitoring.coreos.com/v1 kind: Prometheus metadata: name: prometheus namespace: monitoring spec: replicas: 2 resources: requests: memory: 4Gi serviceAccountName: prometheus serviceMonitorSelector: matchLabels: app: prometheus alerting: alertmanagers: - namespace: monitoring name: alertmanager port: web ruleSelector: matchLabels: prometheus: k8s

2.2 ServiceMonitor配置

apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: app-monitor namespace: monitoring spec: selector: matchLabels: app: my-app endpoints: - port: metrics interval: 30s scrapeTimeout: 10s path: /metrics namespaceSelector: matchNames: - default

2.3 自定义指标暴露

from prometheus_client import start_http_server, Counter, Gauge REQUEST_COUNT = Counter( 'app_requests_total', 'Total number of requests', ['method', 'endpoint'] ) RESPONSE_TIME = Gauge( 'app_response_time_seconds', 'Response time in seconds', ['endpoint'] ) @app.route('/api/users') def get_users(): REQUEST_COUNT.labels(method='GET', endpoint='/api/users').inc() start_time = time.time() users = get_users_from_db() RESPONSE_TIME.labels(endpoint='/api/users').set(time.time() - start_time) return jsonify(users) if __name__ == '__main__': start_http_server(8080) app.run(port=5000)

三、日志管理配置

3.1 Loki部署

apiVersion: loki.grafana.com/v1 kind: LokiStack metadata: name: loki namespace: monitoring spec: size: 1x.small storage: schemas: - version: v13 effectiveDate: "2024-01-01" secret: name: loki-storage tenants: mode: openshift-logging

3.2 Fluentd配置

apiVersion: v1 kind: ConfigMap metadata: name: fluentd-config namespace: logging data: fluent.conf: | <source> @type tail path /var/log/containers/*.log pos_file /var/log/fluentd-containers.log.pos tag kubernetes.* read_from_head true </source> <filter kubernetes.**> @type kubernetes_metadata </filter> <match kubernetes.**> @type loki url https://loki.example.com auth_user admin auth_password secret extra_labels {"cluster": "production"} </match>

3.3 应用日志配置

const winston = require('winston'); const LokiTransport = require('winston-loki'); const logger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', format: winston.format.json(), transports: [ new winston.transports.Console(), new LokiTransport({ host: 'http://loki:3100', labels: { service: 'my-app', environment: 'production' }, json: true }) ] }); logger.info('Application started', { service: 'my-app', version: '1.0.0', timestamp: new Date().toISOString() });

四、分布式追踪配置

4.1 Jaeger部署

apiVersion: jaegertracing.io/v1 kind: Jaeger metadata: name: jaeger namespace: observability spec: strategy: production collector: replicas: 3 query: replicas: 2 storage: type: elasticsearch options: es: server-urls: http://elasticsearch:9200

4.2 OpenTelemetry配置

apiVersion: opentelemetry.io/v1alpha1 kind: OpenTelemetryCollector metadata: name: otel-collector namespace: observability spec: config: | receivers: otlp: protocols: grpc: http: processors: batch: memory_limiter: check_interval: 1s limit_mib: 4000 spike_limit_mib: 8000 exporters: jaeger: endpoint: jaeger:14250 tls: insecure: true service: pipelines: traces: receivers: [otlp] processors: [memory_limiter, batch] exporters: [jaeger]

4.3 应用追踪配置

package main import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/jaeger" "go.opentelemetry.io/otel/sdk/resource" "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.7.0" ) func initTracer() { exporter, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint("http://jaeger:14268/api/traces"))) if err != nil { log.Fatal(err) } tp := trace.NewTracerProvider( trace.WithBatcher(exporter), trace.WithResource(resource.NewWithAttributes( semconv.ServiceNameKey.String("my-app"), )), ) otel.SetTracerProvider(tp) } func main() { initTracer() tracer := otel.Tracer("my-app") ctx, span := tracer.Start(context.Background(), "main") defer span.End() // ... 业务逻辑 }

五、告警配置

5.1 Alertmanager配置

apiVersion: monitoring.coreos.com/v1 kind: Alertmanager metadata: name: alertmanager namespace: monitoring spec: replicas: 2 serviceAccountName: alertmanager config: global: resolve_timeout: 5m route: group_by: ['alertname'] group_wait: 10s group_interval: 10s repeat_interval: 1h receiver: 'webhook' receivers: - name: 'webhook' webhook_configs: - url: 'http://alert-webhook:8080/webhook'

5.2 告警规则配置

apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: application-alerts namespace: monitoring spec: groups: - name: application.rules rules: - alert: HighErrorRate expr: sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.1 for: 5m labels: severity: critical annotations: summary: "High error rate detected" description: "Error rate is {{ $value }}% for service {{ $labels.service }}" - alert: HighLatency expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)) > 2 for: 5m labels: severity: warning annotations: summary: "High latency detected" description: "P95 latency is {{ $value }}s for service {{ $labels.service }}"

六、可视化配置

6.1 Grafana部署

apiVersion: grafana.integreatly.org/v1beta1 kind: Grafana metadata: name: grafana namespace: monitoring spec: config: log: mode: "console" auth: disable_login_form: false datasources: - name: Prometheus type: prometheus access: proxy url: http://prometheus:9090 - name: Loki type: loki access: proxy url: http://loki:3100 - name: Jaeger type: jaeger access: proxy url: http://jaeger:16686 dashboardLabelSelector: matchLabels: app: grafana

6.2 自定义仪表盘

{ "title": "Application Metrics", "panels": [ { "type": "graph", "title": "Request Rate", "targets": [ { "expr": "sum(rate(http_requests_total[5m]))", "legendFormat": "Requests/sec" } ] }, { "type": "singlestat", "title": "Error Rate", "targets": [ { "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) / sum(rate(http_requests_total[5m])) * 100", "legendFormat": "Error %" } ] }, { "type": "graph", "title": "Response Time", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P95" } ] } ] }