当前位置：首页 > news >正文

Kubernetes和机器学习工作负载：从分布式训练到模型服务的全面解析

news 2026/4/29 1:51:23

Kubernetes和机器学习工作负载：从分布式训练到模型服务的全面解析

🔥 硬核开场

各位技术大佬们，今天咱们来聊聊Kubernetes和机器学习工作负载。别以为机器学习只是在单机上跑模型，在云原生时代，Kubernetes已经成为运行机器学习工作负载的最佳平台！今天susu就带你们深入解析Kubernetes上的机器学习工作负载，从分布式训练到模型服务，从GPU调度到资源管理，全给你整明白！

📋 核心内容

1. Kubernetes上运行机器学习的优势

弹性伸缩：根据工作负载自动调整资源
资源隔离：确保不同工作负载之间的资源隔离
标准化部署：使用容器化技术，确保环境一致性
高可用性：支持多副本和故障转移
集成生态：与CI/CD、监控等工具集成

2. 分布式训练

2.1 TensorFlow on Kubernetes

apiVersion: kubeflow.org/v1 kind: TFJob metadata: name: tf-distributed-training spec: tfReplicaSpecs: Master: replicas: 1 template: spec: containers: - name: tensorflow image: tensorflow/tensorflow:2.8.0-gpu command: - python - /app/train.py resources: limits: nvidia.com/gpu: 1 Worker: replicas: 3 template: spec: containers: - name: tensorflow image: tensorflow/tensorflow:2.8.0-gpu command: - python - /app/train.py resources: limits: nvidia.com/gpu: 1 PS: replicas: 2 template: spec: containers: - name: tensorflow image: tensorflow/tensorflow:2.8.0 command: - python - /app/train.py resources: limits: cpu: 2 memory: 4Gi

2.2 PyTorch on Kubernetes

apiVersion: kubeflow.org/v1 kind: PyTorchJob metadata: name: pytorch-distributed-training spec: pytorchReplicaSpecs: Master: replicas: 1 template: spec: containers: - name: pytorch image: pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime command: - python - /app/train.py resources: limits: nvidia.com/gpu: 1 Worker: replicas: 3 template: spec: containers: - name: pytorch image: pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime command: - python - /app/train.py resources: limits: nvidia.com/gpu: 1

2.3 Kubeflow安装

# 安装Kubeflow curl -s https://raw.githubusercontent.com/kubeflow/kfctl/v1.2.0/kfctl.sh | bash # 配置Kubeflow export KF_NAME=my-kubeflow export BASE_DIR=/home/$USER/kubeflow export KF_DIR=${BASE_DIR}/${KF_NAME} mkdir -p ${KF_DIR} cd ${KF_DIR} kfctl apply -V -f https://github.com/kubeflow/manifests/raw/v1.2.0/kfdef/kfctl_k8s_istio.v1.2.0.yaml # 验证安装 kubectl get pods -n kubeflow

3. 模型服务

3.1 TensorFlow Serving

apiVersion: apps/v1 kind: Deployment metadata: name: tensorflow-serving spec: replicas: 3 selector: matchLabels: app: tensorflow-serving template: metadata: labels: app: tensorflow-serving spec: containers: - name: tensorflow-serving image: tensorflow/serving:2.8.0 ports: - containerPort: 8500 - containerPort: 8501 volumeMounts: - name: model-volume mountPath: /models args: - --model_name=my-model - --model_base_path=/models/my-model volumes: - name: model-volume persistentVolumeClaim: claimName: model-pvc --- apiVersion: v1 kind: Service metadata: name: tensorflow-serving spec: selector: app: tensorflow-serving ports: - port: 8500 targetPort: 8500 - port: 8501 targetPort: 8501 type: LoadBalancer

3.2 TorchServe

apiVersion: apps/v1 kind: Deployment metadata: name: torchserve spec: replicas: 3 selector: matchLabels: app: torchserve template: metadata: labels: app: torchserve spec: containers: - name: torchserve image: pytorch/torchserve:0.4.0 ports: - containerPort: 8080 - containerPort: 8081 - containerPort: 8082 volumeMounts: - name: model-volume mountPath: /model-store env: - name: MODEL_NAME value: my-model - name: MODEL_STORE value: /model-store volumes: - name: model-volume persistentVolumeClaim: claimName: model-pvc --- apiVersion: v1 kind: Service metadata: name: torchserve spec: selector: app: torchserve ports: - port: 8080 targetPort: 8080 - port: 8081 targetPort: 8081 - port: 8082 targetPort: 8082 type: LoadBalancer

3.3 Seldon Core

# 安装Seldon Core helm repo add seldon https://charts.seldon.io helm repo update helm install seldon-core seldon/seldon-core-operator --namespace seldon-system --create-namespace # 部署模型 kubectl apply -f - <<EOF apiVersion: machinelearning.seldon.io/v1 kind: SeldonDeployment metadata: name: model-deployment namespace: default spec: predictors: - name: model replicas: 3 graph: name: model implementation: TENSORFLOW_SERVER modelUri: gs://my-bucket/models/my-model endpoints: - type: REST port: 8501 - type: GRPC port: 8500 EOF

4. GPU管理

4.1 GPU节点配置

# 安装NVIDIA驱动 curl -O https://us.download.nvidia.com/XFree86/Linux-x86_64/470.57.02/NVIDIA-Linux-x86_64-470.57.02.run chmod +x NVIDIA-Linux-x86_64-470.57.02.run ./NVIDIA-Linux-x86_64-470.57.02.run --silent # 安装NVIDIA Container Toolkit distribution=$(. /etc/os-release;echo $ID$VERSION_ID) curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit sudo systemctl restart docker # 验证GPU nvidia-smi # 标记节点 kubectl label nodes <node-name> hardware-type=NVIDIA-GPU

4.2 GPU资源调度

apiVersion: v1 kind: Pod metadata: name: gpu-pod spec: containers: - name: gpu-container image: tensorflow/tensorflow:2.8.0-gpu resources: limits: nvidia.com/gpu: 1 command: - bash - -c - | nvidia-smi python -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"

4.3 GPU监控

# 安装NVIDIA GPU监控 helm repo add gpu-operator https://nvidia.github.io/gpu-operator helm repo update helm install gpu-operator gpu-operator/gpu-operator --namespace gpu-operator --create-namespace # 查看GPU使用情况 kubectl get pods -n gpu-operator kubectl logs -n gpu-operator deployment/gpu-operator

5. 数据管理

5.1 数据存储

apiVersion: v1 kind: PersistentVolumeClaim metadata: name:># 创建数据预处理Job kubectl apply -f - <<EOF apiVersion: batch/v1 kind: Job metadata: name:># 安装Katib kubectl apply -f https://raw.githubusercontent.com/kubeflow/katib/master/manifests/v1beta1/installs/katib-standalone.yaml # 创建超参数调优实验 kubectl apply -f - <<EOF apiVersion: kubeflow.org/v1beta1 kind: Experiment metadata: name: hyperparameter-tuning namespace: kubeflow spec: objective: type: maximize goal: 0.99 objectiveMetricName: accuracy additionalMetricNames: - loss algorithm: algorithmName: random parallelTrialCount: 3 maxTrialCount: 12 maxFailedTrialCount: 3 parameters: - name: learning_rate parameterType: double feasibleSpace: min: "0.001" max: "0.1" - name: batch_size parameterType: int feasibleSpace: min: "32" max: "256" trialTemplate: goTemplate: rawTemplate: |- apiVersion: batch/v1 kind: Job metadata: name: {{.TrialName}} namespace: {{.NameSpace}} spec: template: spec: containers: - name: training image: my-training:latest command: - python - /app/train.py - --learning_rate={{.HyperParameters.learning_rate}} - --batch_size={{.HyperParameters.batch_size}} restartPolicy: Never EOF # 查看实验状态 kubectl get experiment -n kubeflow kubectl describe experiment hyperparameter-tuning -n kubeflow

6.2 模型版本管理

# 安装MLflow helm repo add mlflow https://mlflow.github.io/helm-charts helm repo update helm install mlflow mlflow/mlflow --namespace mlflow --create-namespace # 访问MLflow UI kubectl port-forward svc/mlflow 5000:5000 -n mlflow

7. 监控与可观测性

7.1 模型性能监控

apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: model-serving namespace: monitoring spec: selector: matchLabels: app: tensorflow-serving endpoints: - port: 8501 interval: 15s path: /v1/models/my-model/metrics

7.2 训练作业监控

apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: name: training-job namespace: monitoring spec: selector: matchLabels: app: training-job endpoints: - port: metrics interval: 15s

8. 安全最佳实践

8.1 访问控制

apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: ml-operator namespace: kubeflow rules: - apiGroups: ["kubeflow.org"] resources: ["tfjobs", "pytorchjobs"] verbs: ["get", "list", "create", "update", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: ml-operator-binding namespace: kubeflow subjects: - kind: ServiceAccount name: ml-operator namespace: kubeflow roleRef: kind: Role name: ml-operator apiGroup: rbac.authorization.k8s.io

8.2 数据安全

apiVersion: v1 kind: Secret metadata: name:>apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: mljobs.example.com spec: group: example.com versions: - name: v1 served: true storage: true schema: openAPIV3Schema: type: object properties: spec: type: object properties: modelName: type: string trainingConfig: type: object properties: epochs: type: integer batchSize: type: integer servingConfig: type: object properties: replicas: type: integer scope: Namespaced names: plural: mljobs singular: mljob kind: MLJob shortNames: - mlj

9.2 操作符开发

// main.go package main import ( "flag" "fmt" "os" "os/signal" "syscall" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/example/ml-operator/controllers" mlv1 "github.com/example/ml-operator/pkg/apis/ml/v1" ) func main() { var namespace string flag.StringVar(&namespace, "namespace", "", "Namespace to watch for MLJob resources") flag.Parse() mgr, err := manager.New(cache.Options{ Namespace: namespace, }, manager.Options{ MetricsBindAddress: ":8080", }) if err != nil { fmt.Fprintf(os.Stderr, "Error creating manager: %v\n", err) os.Exit(1) } if err := mlv1.AddToScheme(mgr.GetScheme()); err != nil { fmt.Fprintf(os.Stderr, "Error adding MLJob scheme: %v\n", err) os.Exit(1) } if err := controllers.AddMLJobController(mgr); err != nil { fmt.Fprintf(os.Stderr, "Error adding MLJob controller: %v\n", err) os.Exit(1) } stopCh := signals.SetupSignalHandler() if err := mgr.Start(stopCh); err != nil { fmt.Fprintf(os.Stderr, "Error starting manager: %v\n", err) os.Exit(1) } }