微服务调用链路：

用户请求 → Service A → Service B → Service C
                              ↓
                         Service D

问题：
- 请求经过了哪些服务？
- 每个服务耗时多久？
- 哪个环节出现了错误？

链路追踪解决：
┌─────────────────────────────────────────────────────────┐
│ Trace: 用户请求                                         │
│ ├─ Span: Service A (100ms)                              │
│ │  ├─ Span: Service B (50ms)                            │
│ │  │  └─ Span: Service C (30ms)                         │
│ │  └─ Span: Service D (40ms)                            │
└─────────────────────────────────────────────────────────┘

OpenTelemetry = Traces + Metrics + Logs

┌─────────────────────────────────────────────────────────┐
│                 OpenTelemetry 架构                       │
│                                                         │
│  ┌──────────────────────────────────────────────────┐  │
│  │              应用程序                             │  │
│  │  ┌─────────────────────────────────────────────┐ │  │
│  │  │          OpenTelemetry SDK                   │ │  │
│  │  │  TracerProvider + MeterProvider              │ │  │
│  │  └──────────────────┬──────────────────────────┘ │  │
│  └─────────────────────┼────────────────────────────┘  │
│                        │                                │
│  ┌─────────────────────┼────────────────────────────┐  │
│  │                     ▼                             │  │
│  │         OpenTelemetry Collector                   │  │
│  │    接收、处理、导出遥测数据                        │  │
│  └─────────────────────┬────────────────────────────┘  │
│                        │                                │
│         ┌──────────────┼──────────────┐               │
│         ▼              ▼              ▼               │
│  ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
│  │    Jaeger    │ │   Zipkin     │ │  Prometheus  │ │
│  └──────────────┘ └──────────────┘ └──────────────┘ │
└─────────────────────────────────────────────────────────┘

# 使用 Helm 部署
helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
helm repo update

# 安装 Jaeger
helm install jaeger jaegertracing/jaeger \
  --namespace tracing \
  --create-namespace

# 查看状态
kubectl get pods -n tracing

# 端口转发
kubectl port-forward svc/jaeger-query 16686:80 -n tracing

# 访问 http://localhost:16686

# otel-collector.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
  namespace: tracing
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318

    processors:
      batch:
        timeout: 5s
        send_batch_size: 1024

    exporters:
      jaeger:
        endpoint: jaeger-collector.tracing:14250
        tls:
          insecure: true

    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [batch]
          exporters: [jaeger]
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
  namespace: tracing
spec:
  replicas: 1
  selector:
    matchLabels:
      app: otel-collector
  template:
    metadata:
      labels:
        app: otel-collector
    spec:
      containers:
      - name: otel-collector
        image: otel/opentelemetry-collector-contrib:latest
        ports:
        - containerPort: 4317
        - containerPort: 4318
        volumeMounts:
        - name: config
          mountPath: /etc/otelcol-contrib/config.yaml
          subPath: config.yaml
      volumes:
      - name: config
        configMap:
          name: otel-collector-config

# 安装依赖
# pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource

# 配置 TracerProvider
resource = Resource.create({
    "service.name": "my-app",
    "service.version": "1.0.0",
})

provider = TracerProvider(resource=resource)
processor = BatchSpanProcessor(OTLPSpanExporter(
    endpoint="otel-collector.tracing:4317",
    insecure=True
))
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# 获取 Tracer
tracer = trace.get_tracer(__name__)

# 创建 Span
with tracer.start_as_current_span("main-operation") as span:
    span.set_attribute("http.method", "GET")
    span.set_attribute("http.url", "/api/users")
    
    # 子 Span
    with tracer.start_as_current_span("database-query") as child_span:
        child_span.set_attribute("db.system", "postgresql")
        # 执行数据库查询

package main

import (
	"context"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/sdk/resource"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
)

func initTracer() (*sdktrace.TracerProvider, error) {
	exporter, err := otlptracegrpc.New(
		context.Background(),
		otlptracegrpc.WithEndpoint("otel-collector.tracing:4317"),
		otlptracegrpc.WithInsecure(),
	)
	if err != nil {
		return nil, err
	}

	tp := sdktrace.NewTracerProvider(
		sdktrace.WithBatcher(exporter),
		sdktrace.WithResource(resource.NewWithAttributes(
			semconv.SchemaURL,
			semconv.ServiceNameKey.String("my-app"),
		)),
	)
	otel.SetTracerProvider(tp)
	return tp, nil
}

1. 发送测试请求
2. 访问 Jaeger UI
3. 选择 Service: my-app
4. 点击 Find Traces
5. 查看链路详情：
   - 调用链路图
   - 各 Span 耗时
   - 错误信息
   - 标签和日志

# 采样配置
sampler:
  type: probabilistic
  param: 0.1  # 采样 10% 的请求

# 其他采样类型
# - always: 全部采样（开发环境）
# - never: 不采样
# - rate_limiting: 限速采样

好的命名：
- HTTP GET /api/users
- db.query SELECT * FROM users
- cache.get user:123

不好的命名：
- span1
- my-span
- operation