This page looks best with JavaScript enabled

Ascend NPU Driver Installation

 ·  ☕ 5 min read

1. Install the Driver

  • Create the HwHiAiUser user
1
2

groupadd -g 1000 HwHiAiUser
useradd -g HwHiAiUser -u 1000 -d /home/HwHiAiUser -m HwHiAiUser -s /bin/bash
  • Add directory permissions
1
2

chown -R HwHiAiUser /usr/local/Ascend
chmod -R 755 /usr/local/Ascend
  • Download the driver and firmware

Go to https://www.hiascend.ru/hardware/firmware-drivers/community?product=1&model=30&cann=All&driver=1.0.26.alpha to find the matching driver and firmware.

1

wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend%20HDK/Ascend%20HDK%2024.1.RC2.2/Ascend-hdk-910b-npu-driver_24.1.rc2.2_linux-x86-64.run

1

wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Ascend%20HDK/Ascend%20HDK%2024.1.RC2.2/Ascend-hdk-910b-npu-firmware_7.3.0.2.220.run
  • Install the driver
1

bash ./Ascend-hdk-910b-npu-driver_24.1.rc2.2_linux-x86-64.run --full --install-for-all
  • Install the firmware
1

bash ./Ascend-hdk-910b-npu-firmware_7.3.0.2.220.run --full

2. Install ascend-docker-runtime

  • Download ascend-docker-runtime

Go to https://gitee.com/ascend/ascend-docker-runtime/releases/tag/v5.0.0-RC3.2 to find the download link for the matching architecture.

1

wget https://gitee.com/ascend/ascend-docker-runtime/releases/download/v5.0.0-RC3.2/Ascend-docker-runtime_5.0.RC3.2_linux-x86_64.run
  • Install ascend-docker-runtime
1

bash ./Ascend-docker-runtime_5.0.RC3.2_linux-x86_64.run --install

3. Install Docker [Optional]

Choose either Docker or Containerd.

3.1 Install docker

  • Add the key
1
2

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - &&
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
  • Add the source
1
2
3
4

cat >> /etc/apt/sources.list.d/docker.list << EOF
deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu xenial stable
deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
EOF
  • Update the sources
1

apt-get update
  • Install docker
1

apt-get install docker-ce=5:20.10.7~3-0~ubuntu-xenial docker-ce-cli=5:20.10.7~3-0~ubuntu-xenial containerd.io -y

3.2 Configure docker

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

cat <<EOF >  /etc/docker/daemon.json
{
  "bip": "x.x.x.x/24",
  "live-restore": true,
  "data-root": "/data/docker",
  "default-runtime": "ascend",
  "insecure-registries": [
  ],
  "registry-mirrors": [
  ],
  "runtimes": {
    "ascend": {
      "path": "/usr/local/Ascend/Ascend-Docker-Runtime/ascend-docker-runtime",
      "runtimeArgs": []
    }
  }
}
EOF
  • Restart docker
1
2

systemctl daemon-reload
systemctl restart docker

3.3 Install cri-docker

Used for Kubelet calls. If you are using containerd, you can skip this step.

  • Install CRI-Docker
1
2
3
4

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.10/cri-dockerd-0.3.10.amd64.tgz
tar -zxvf cri-dockerd-0.3.10.amd64.tgz
cp cri-dockerd/cri-dockerd /usr/local/bin/cri-dockerd
chmod +x /usr/local/bin/cri-dockerd
  • Add the user group
1

groupadd docker
  • Configure the startup file
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

cat <<EOF > /etc/systemd/system/cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify

ExecStart=/usr/local/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9

ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitBurst=3

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF
  • Generate the socket file
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

cat <<EOF > /etc/systemd/system/cri-docker.socket
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF
  • Start CRI-DOCKER
1
2
3
4

systemctl daemon-reload
systemctl start cri-docker
systemctl enable cri-docker
systemctl is-active cri-docker

3.4 Verify

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

docker run --security-opt apparmor=unconfined --security-opt seccomp=unconfined --rm -it --ipc=host \
                    --device=/dev/davinci0 \
                    --device=/dev/davinci1 \
                    --device=/dev/davinci2 \
                    --device=/dev/davinci3 \
                    --device=/dev/davinci_manager \
                    --device=/dev/devmm_svm \
                    --device=/dev/hisi_hdc \
                    -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
                    -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \
                    -v /usr/local/sbin/:/usr/local/sbin/ \
                    -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
                    -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \
                    -v /var/log/npu/slog/:/var/log/npu/slog \
                    -v /var/log/npu/profiling/:/var/log/npu/profiling \
                    -v /var/log/npu/dump/:/var/log/npu/dump \
                    -v /var/log/npu/:/usr/slog \
                    shaowenchen/demo:hccl-test-8.0.RC2-ubuntu22.04 \
                    npu-smi info

4. Install Containerd [Optional]

Choose either Docker or Containerd.

4.1 Install containerd

  • Add the source
1
2

apt-get update
apt-get install -y ca-certificates curl gnupg lsb-release

1
2

mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg

1
2
3

echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
  • Install containerd
1
2

apt update
apt install containerd.io=1.6.31-1

4.2 Configure containerd

  • Generate the containerd config file (toml)
1
2

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
  • Modify the containerd config file
1
2
3
4

sed -i 's#root = "/var/lib/containerd"#root = "/data/containerd"#g' /etc/containerd/config.toml
sed -i 's#state = "/run/containerd"#state = "/data/run/containerd"#g' /etc/containerd/config.toml
sed -i 's#sandbox_image = "registry.k8s.io/pause:3.6"#sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"#g' /etc/containerd/config.toml
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml
  • Change the runtime to ascend
1

sed -i 's#runtime = "runc"#runtime = "/usr/local/Ascend/Ascend-Docker-Runtime/ascend-docker-runtime"#g' /etc/containerd/config.toml
  • Restart containerd
1

systemctl restart containerd

4.3 Verify

  • Install the nerdctl tool
1

opscli task -f ~/.ops/tasks/install-nerdctl.yaml
  • Verify

ARM image

1

export TEST_IMAGE=shaowenchen/python38-cann:8.0rc3-arm64

AMD64 image

1

export TEST_IMAGE=shaowenchen/python38-cann:8.0rc3-amd64

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

nerdctl -n k8s.io run --rm -it --ipc=host \
                    --device=/dev/davinci0 \
                    --device=/dev/davinci1 \
                    --device=/dev/davinci2 \
                    --device=/dev/davinci3 \
                    --device=/dev/davinci_manager \
                    --device=/dev/devmm_svm \
                    --device=/dev/hisi_hdc \
                    -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
                    -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \
                    -v /usr/local/sbin/:/usr/local/sbin/ \
                    -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
                    -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \
                    -v /var/log/npu/slog/:/var/log/npu/slog \
                    -v /var/log/npu/profiling/:/var/log/npu/profiling \
                    -v /var/log/npu/dump/:/var/log/npu/dump \
                    -v /var/log/npu/:/usr/slog \
                    ${TEST_IMAGE} \
                    npu-smi info

4. Join the K8s Cluster

4.1 Change the Hostname

1
2

export HOSTNAME=k8s-ascend-910b-27
hostnamectl set-hostname ${HOSTNAME}

Create the working directory for the device management plugin

1

mkdir -p /var/log/mindx-dl/devicePlugin

4.2 Initialize Kernel Parameters

1

opscli task -f ~/.ops/task/set-host.yaml

4.3 Install the K8s Base Components

  • Add the K8s source

https://developer.aliyun.com/mirror/kubernetes/ — add it for versions below 1.28

1

curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -

1
2
3

cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

1

apt-get update
  • Install the K8s base components
1

export K8S_VERSION=1.25.6

1

apt-get install kubeadm=${K8S_VERSION}-00 kubelet=${K8S_VERSION}-00 kubectl=${K8S_VERSION}-00 -y

4.4 Join the Cluster

  • Generate a Token

Generate the token on the master node

1

kubeadm token create --print-join-command
  • Join the cluster

If the worker node uses the Docker Runtime, you need to add the parameter --cri-socket unix:///var/run/cri-dockerd.sock when joining the cluster.

1

kubeadm join x.x.x.x:6443 --token x.x --discovery-token-ca-cert-hash sha256:x --cri-socket unix:///var/run/cri-dockerd.sock

4.5 Create a Test Pod

  • Create the Pod
1

export TEST_IMAGE=shaowenchen/python38-cann:8.0rc3-arm64

AMD64 image

1

export TEST_IMAGE=shaowenchen/python38-cann:8.0rc3-amd64

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

kubectl apply -f - <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: npu-demo
spec:
  nodeName: ${HOSTNAME}
  containers:
    - name: npu-demo
      image: ${TEST_IMAGE}
      command: ["npu-smi", "info"]
      resources:
        requests:
          huawei.com/Ascend910: 4
        limits:
          huawei.com/Ascend910: 4
EOF
  • Check the Pod status
1

kubectl logs npu-demo
  • Delete the Pod
1

kubectl delete pod npu-demo

5. NPU Status Checks

  • Disable TLS
1

for i in {0..15};do hccn_tool -i $i -tls -g| grep 'switch';done
  • IP
1

for i in {0..15};do hccn_tool  -i $i  -ip -g;done
  • Gateway
1

for i in {0..15};do hccn_tool  -i $i  -gateway -g;done
  • Health status
1

for i in {0..15};do npu-smi info -t health -i $i -c 0| grep 'Health Status';done
  • Link status
1

for i in {0..15}; do hccn_tool -i $i -link -g;done
  • NIC status
1

for i in {0..15};do hccn_tool -i $i -net_health -g;done
  • ECC
1

for i in {0..15}; do npu-smi info -t ecc -i $i;done

6. References

微信公众号
WRITTEN BY
微信公众号

See Also