apiVersion: workload.serving.volcano.sh/v1alpha1
kind: ModelServing
metadata:
  name: deepseek-v2-lite
  namespace: dev
spec:
  schedulerName: volcano
  replicas: 1
  recoveryPolicy: ServingGroupRecreate
  template:
    restartGracePeriodSeconds: 60
    roles:
      - name: prefill
        replicas: 1
        entryTemplate:
          spec:
            initContainers:
              - name: downloader
                imagePullPolicy: Always
                image: ghcr.io/kthena/downloader:latest
                args:
                  - --source
                  - deepseek-ai/DeepSeek-V2-Lite
                  - --output-dir
                  - /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                volumeMounts:
                  - name: models
                    mountPath: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
            containers:
              - name: runtime
                image: ghcr.io/kthena/runtime:latest
                ports:
                  - containerPort: 8100
                args:
                  - --port
                  - "8100"
                  - --engine
                  - vllm
                  - --pod
                  - $(POD_NAME).$(NAMESPACE)
                  - --model
                  - deepseek-v2-lite
                  - --engine-base-url
                  - http://localhost:8000
              - name: vllm
                image: ghcr.io/kthena/kthena-engine:vllm-ascend_v0.10.1rc1_mooncake_v0.3.5
                ports:
                  - containerPort: 8000
                env:
                  - name: HF_HUB_OFFLINE
                    value: "1"
                  - name: HCCL_IF_IP
                    valueFrom:
                      fieldRef:
                        fieldPath: status.podIP
                  - name: GLOO_SOCKET_IFNAME
                    value: eth0
                  - name: TP_SOCKET_IFNAME
                    value: eth0
                  - name: HCCL_SOCKET_IFNAME
                    value: eth0
                  - name: VLLM_LOGGING_LEVEL
                    value: DEBUG
                  - name: AscendRealDevices
                    valueFrom:
                      fieldRef:
                        fieldPath: metadata.annotations['huawei.com/AscendReal']
                args:
                  - "/mnt/cache/deepseek-ai/DeepSeek-V2-Lite/"
                  - "--served-model-name"
                  - "deepseek-ai/DeepSeekV2"
                  - "--tensor-parallel-size"
                  - "2"
                  - "--gpu-memory-utilization"
                  - "0.8"
                  - "--max-model-len"
                  - "8192"
                  - "--max-num-batched-tokens"
                  - "8192"
                  - "--trust-remote-code"
                  - "--enforce-eager"
                  - "--kv-transfer-config"
                  - '{"kv_connector":"MooncakeConnectorV1","kv_buffer_device":"npu","kv_role":"kv_producer","kv_parallel_size":1,"kv_port":"20001","engine_id":"0","kv_rank":0,"kv_connector_module_path":"vllm_ascend.distributed.mooncake_connector","kv_connector_extra_config":{"prefill":{"dp_size":2,"tp_size":2},"decode":{"dp_size":2,"tp_size":2}}}'
                imagePullPolicy: Always
                resources:
                  limits:
                    cpu: "8"
                    memory: 64Gi
                    huawei.com/ascend-1980: "4"
                  requests:
                    cpu: "8"
                    memory: 64Gi
                    huawei.com/ascend-1980: "4"
                readinessProbe:
                  initialDelaySeconds: 5
                  periodSeconds: 5
                  failureThreshold: 3
                  httpGet:
                    path: /health
                    port: 8000
                livenessProbe:
                  initialDelaySeconds: 900
                  periodSeconds: 5
                  failureThreshold: 3
                  httpGet:
                    path: /health
                    port: 8000
                volumeMounts:
                  - name: models
                    mountPath: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                    readOnly: true
                  - name: hccn-config
                    mountPath: /etc/hccn.conf
                    readOnly: true
                  - name: shared-memory-volume
                    mountPath: /dev/shm
            volumes:
              - name: models
                hostPath:
                  path: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                  type: DirectoryOrCreate
              - name: hccn-config
                hostPath:
                  path: /etc/hccn.conf
                  type: File
              - name: shared-memory-volume
                emptyDir:
                  sizeLimit: 256Mi
                  medium: Memory
        workerReplicas: 0
        workerTemplate:
          spec:
            containers: []
      - name: decode
        replicas: 1
        entryTemplate:
          spec:
            initContainers:
              - name: downloader
                imagePullPolicy: Always
                image: ghcr.io/kthena/downloader:latest
                args:
                  - --source
                  - deepseek-ai/DeepSeek-V2-Lite
                  - --output-dir
                  - /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                volumeMounts:
                  - name: models
                    mountPath: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
            containers:
              - name: runtime
                image: ghcr.io/kthena/runtime:latest
                ports:
                  - containerPort: 8100
                args:
                  - --port
                  - "8100"
                  - --engine
                  - vllm
                  - --pod
                  - $(POD_NAME).$(NAMESPACE)
                  - --model
                  - deepseek-v2-lite
                  - --engine-base-url
                  - http://localhost:8000
              - name: vllm
                image: ghcr.io/kthena/kthena-engine:vllm-ascend_v0.10.1rc1_mooncake_v0.3.5
                ports:
                  - containerPort: 8000
                env:
                  - name: HF_HUB_OFFLINE
                    value: "1"
                  - name: HCCL_IF_IP
                    valueFrom:
                      fieldRef:
                        fieldPath: status.podIP
                  - name: GLOO_SOCKET_IFNAME
                    value: eth0
                  - name: TP_SOCKET_IFNAME
                    value: eth0
                  - name: HCCL_SOCKET_IFNAME
                    value: eth0
                  - name: VLLM_LOGGING_LEVEL
                    value: DEBUG
                  - name: AscendRealDevices
                    valueFrom:
                      fieldRef:
                        fieldPath: metadata.annotations['huawei.com/AscendReal']
                args:
                  - "/mnt/cache/deepseek-ai/DeepSeek-V2-Lite/"
                  - "--served-model-name"
                  - "deepseek-ai/DeepSeekV2"
                  - "--tensor-parallel-size"
                  - "2"
                  - "--gpu-memory-utilization"
                  - "0.8"
                  - "--max-model-len"
                  - "8192"
                  - "--max-num-batched-tokens"
                  - "16384"
                  - "--trust-remote-code"
                  - "--no-enable-prefix-caching"
                  - "--enforce-eager"
                  - "--kv-transfer-config"
                  - '{"kv_connector":"MooncakeConnectorV1","kv_buffer_device":"npu","kv_role":"kv_consumer","kv_parallel_size":1,"kv_port":"20002","engine_id":"1","kv_rank":1,"kv_connector_module_path":"vllm_ascend.distributed.mooncake_connector","kv_connector_extra_config":{"prefill":{"dp_size":2,"tp_size":2},"decode":{"dp_size":2,"tp_size":2}}}'
                imagePullPolicy: Always
                resources:
                  limits:
                    cpu: "8"
                    memory: 64Gi
                    huawei.com/ascend-1980: "4"
                  requests:
                    cpu: "8"
                    memory: 64Gi
                    huawei.com/ascend-1980: "4"
                readinessProbe:
                  initialDelaySeconds: 5
                  periodSeconds: 5
                  failureThreshold: 3
                  httpGet:
                    path: /health
                    port: 8000
                livenessProbe:
                  initialDelaySeconds: 900
                  periodSeconds: 5
                  failureThreshold: 3
                  httpGet:
                    path: /health
                    port: 8000
                volumeMounts:
                  - name: models
                    mountPath: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                    readOnly: true
                  - name: hccn-config
                    mountPath: /etc/hccn.conf
                    readOnly: true
                  - name: shared-memory-volume
                    mountPath: /dev/shm
            volumes:
              - name: models
                hostPath:
                  path: /mnt/cache/deepseek-ai/DeepSeek-V2-Lite/
                  type: DirectoryOrCreate
              - name: hccn-config
                hostPath:
                  path: /etc/hccn.conf
                  type: File
              - name: shared-memory-volume
                emptyDir:
                  sizeLimit: 256Mi
                  medium: Memory
        workerReplicas: 0
        workerTemplate:
          spec:
            containers: []
