1. Building the Image
To make testing convenient, the model files are packaged into the image here.
1
2
3
4
| git clone https://huggingface.co/Qwen/Qwen1.5-1.8B-Chat
cd Qwen1.5-1.8B-Chat && git lfs pull
rm -rf .git
cd ..
|
1
2
3
4
5
| cat <<EOF > Dockerfile
FROM vllm/vllm-openai:latest
RUN mkdir -p /models/Qwen1.5-1.8B-Chat
COPY Qwen1.5-1.8B-Chat/* /models/Qwen1.5-1.8B-Chat
EOF
|
1
| nerdctl build --platform=amd64 -t registry-1.docker.io/shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64 .
|
1
| nerdctl push --platform=amd64 registry-1.docker.io/shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64
|
2. Inference Service on the Host
- Set environment variables
China
1
| export IMAGE=shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64
|
Overseas
1
| export IMAGE=registry-1.docker.io/shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64
|
- Specify the device and run the service
1
2
3
4
5
6
7
8
9
| nerdctl run --gpus "device=1" \
--security-opt apparmor=unconfined \
--security-opt seccomp=unconfined \
-p 8000:8000 \
--name Qwen1.5-1.8B-Chat-allinone \
--ipc=host \
$IMAGE \
--model /models/Qwen1.5-1.8B-Chat \
--dtype=half
|
- Test the inference endpoint
1
2
3
4
5
6
7
8
9
| curl http://127.0.0.1:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "/models/Qwen1.5-1.8B-Chat",
"messages": [
{"role": "user", "content": "什么是大模型"}
],
"max_tokens": 1024
}'
|
1
| nerdctl rm Qwen1.5-1.8B-Chat-allinone
|
3. Inference Service on the Cluster
- Set environment variables
China
1
| export IMAGE=shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64
|
Overseas
1
| export IMAGE=registry-1.docker.io/shaowenchen/demo:vllm-qwen-1.5-1.8b-chat-amd64
|
Set the node to run on
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
| kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: demo-vllm-qwen1.5-1.8b-chat
namespace: default
spec:
replicas: 1
selector:
matchLabels:
app: demo-vllm-qwen1.5-1.8b-chat
template:
metadata:
labels:
app: demo-vllm-qwen1.5-1.8b-chat
spec:
nodeName: $NODE_NAME
containers:
- name: demo-vllm-qwen
image: $IMAGE
args:
- "--dtype"
- "half"
- "--model"
- "/models/Qwen1.5-1.8B-Chat"
EOF
|
- Test the inference endpoint
1
| kubectl exec -it deployment/demo-vllm-qwen1.5-1.8b-chat -- bash
|
1
2
3
4
5
6
7
8
9
| curl http://127.0.0.1:8000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "/models/Qwen1.5-1.8B-Chat",
"messages": [
{"role": "user", "content": "什么是大模型"}
],
"max_tokens": 1024
}'
|
1
| kubectl delete deployment demo-vllm-qwen1.5-1.8b-chat
|
