使用 Volcano 运行 hccl-test

1. 制作 hccl-test 镜像

下载依赖包

Python-3.8.18.tgz
Ascend-cann-toolkit_8.0.RC2_linux-x86_64.run
Ascend-cann-kernels-910b_8.0.RC2_linux.run
mpich-3.2.1.tar.gz

如果不方便下载，也可以直接从我打包的镜像中拷贝出来。

编写 Dockerfile

FROM ubuntu:22.04

WORKDIR /home

RUN sed -i -e 's/^APT/# APT/' -e 's/^DPkg/# DPkg/' /etc/apt/apt.conf.d/docker-clean
RUN apt-get update && apt-get install -y apt-transport-https tzdata openssh-server

# Install base library
RUN apt-get install -y gcc g++ make cmake zlib1g zlib1g-dev openssl libsqlite3-dev libssl-dev libffi-dev libbz2-dev libxslt1-dev unzip pciutils net-tools libblas-dev gfortran libblas3 liblzma-dev git wget zip vim gnutls-bin

# Install Python3.8.18
COPY Python-3.8.18.tgz .
RUN tar -zxvf Python-3.8.18.tgz \
    && cd Python-3.8.18 \
    && chmod +x ./configure
RUN cd Python-3.8.18 && ./configure --prefix=/usr/local/python3.8.18 --enable-loadable-sqlite-extensions --enable-shared  --enable-optimizations
RUN cd Python-3.8.18 && make -j16 && make install
RUN cd /home \
    && ln -s /usr/local/python3.8.18/bin/pip3 /usr/bin/pip \
    && ln -s /usr/local/python3.8.18/bin/python3 /usr/bin/python
ENV LD_LIBRARY_PATH=/usr/local/python3.8.18/lib:$LD_LIBRARY_PATH
ENV PATH=/usr/local/python3.8.18/bin:$PATH

# Install ascend-toolkit & kernels
COPY Ascend-cann-toolkit_8.*.run .
COPY Ascend-cann-kernels-910b_8.*.run .
RUN ./Ascend-cann-toolkit_8.*.run --install --quiet
RUN ./Ascend-cann-kernels-910b_8.*.run --install --quiet
RUN echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >> ~/.bashrc

# Set Env
ENV TOOLKIT_PATH=/usr/local/Ascend/ascend-toolkit/latest
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/driver/:$LD_LIBRARY_PATH

ENV GLOG_v=2 \
    LD_LIBRARY_PATH=$TOOLKIT_PATH/lib64:$LD_LIBRARY_PATH \
    TBE_IMPL_PATH=$TOOLKIT_PATH/opp/op_impl/built-in/ai_core/tbe \
    PATH=$TOOLKIT_PATH/ccec_compiler/bin:$PATH \
    ASCEND_OPP_PATH=$TOOLKIT_PATH/opp \
    ASCEND_AICPU_PATH=$TOOLKIT_PATH

ENV PYTHONPATH=$TBE_IMPL_PATH:$PYTHONPATH
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/mpich-3.2.1/lib:$TOOLKIT_PATH/lib64:$TOOLKIT_PATH/x86_64-linux/devlib

# Install hccl_test
COPY mpich-3.2.1.tar.gz .
RUN umask 0022 && \
    tar -zxvf mpich-3.2.1.tar.gz && \
    cd mpich-3.2.1 && \
    ./configure --disable-fortran --prefix=/usr/local/mpich-3.2.1 && \
    make && \
    make install && \
    cd $TOOLKIT_PATH/tools/hccl_test && \
    make MPI_HOME=/usr/local/mpich-3.2.1 ASCEND_DIR=$TOOLKIT_PATH/x86_64-linux

ENV PATH=$PATH:/usr/local/mpich-3.2.1/bin:$TOOLKIT_PATH/tools/hccl_test/bin

编译 hccl-test 镜像

`1`	`docker build -t shaowenchen/hccl-test:8.0.RC2-ubuntu22.04 -f Dockerfile .`

推送 hccl-test 镜像

`1`	`docker push shaowenchen/hccl-test:8.0.RC2-ubuntu22.04`

2. 运行 Volcano Job

给测试节点打上标签

1
2
3

kubectl label node ascend-910b-01 hccl-test=true
kubectl label node ascend-910b-02 hccl-test=true
kubectl label node ascend-910b-03 hccl-test=true

创建 Volcano Job

1
2

export HCCL_TEST_IMAGE=shaowenchen/hccl-test:8.0.RC2-ubuntu22.04
export HCCL_TEST_NODES=3

cat <<EOF | kubectl apply -f -
apiVersion: batch.volcano.sh/v1alpha1
kind: Job
metadata:
  name: hccl-test
spec:
  minAvailable: $HCCL_TEST_NODES
  schedulerName: volcano
  queue: default
  priorityClassName: extreme-high-priority-preempting
  policies:
    - event: PodEvicted
      action: RestartJob
  plugins:
    env: []
    svc: []
    ssh: []
  tasks:
    - replicas: $HCCL_TEST_NODES
      name: hccl-test
      policies:
      - event: TaskCompleted
        action: CompleteJob
      template:
        metadata:
          labels:
            app: hccl-test
        spec:
          containers:
            - command:
              - /bin/sh
              - -c
              - |
                mkdir -p /var/run/sshd; /usr/sbin/sshd;
                sleep infinity                
              image: $HCCL_TEST_IMAGE
              imagePullPolicy: Always
              name: hccl-test
              resources:
                requests:
                  cpu: 1
                  huawei.com/Ascend910: 16
                limits:
                  huawei.com/Ascend910: 16
          nodeSelector:
            hccl-test: "true"
          affinity:
            podAntiAffinity:
              requiredDuringSchedulingIgnoredDuringExecution:
                - labelSelector:
                    matchExpressions:
                     - key: app
                       operator: In
                       values:
                       - hccl-test
                  topologyKey: "kubernetes.io/hostname"
EOF

3. 运行 hccl-test 测试用例

查看 Pod 状态

kubectl get pod -l app=hccl-test -o wide

NAME                    READY   STATUS    RESTARTS   AGE     IP              NODE             NOMINATED NODE   READINESS GATES
hccl-test-hccl-test-0   1/1     Running   0          7m19s   10.244.53.163   ascend-910b-02   <none>           <none>
hccl-test-hccl-test-1   1/1     Running   0          7m19s   10.244.54.38    ascend-910b-01   <none>           <none>
hccl-test-hccl-test-2   1/1     Running   0          7m19s   10.244.52.160   ascend-910b-03   <none>           <none>

进入 Pod

`1`	`kubectl exec -it hccl-test-hccl-test-0 bash`

单机测试

export HCCL_RDMA_TC=100
export HCCL_RDMA_SL=3
export HCCL_BUFFSIZE=2048
export HCCL_PROC=16
mpirun -n ${HCCL_PROC} /usr/local/Ascend/ascend-toolkit/latest/tools/hccl_test/bin/all_reduce_test -b 8K -e 64M -f 2 -d fp32 -o sum -p ${HCCL_PROC}

多节点

export HCCL_RDMA_TC=100
export HCCL_RDMA_SL=3
export HCCL_BUFFSIZE=2048

mpirun -n 32 -hosts hccl-test-hccl-test-0.hccl-test:16,hccl-test-hccl-test-0.hccl-test:16,hccl-test-hccl-test-0.hccl-test:16 all_reduce_test -b 8K -e 4G -f 2 -d fp32 -o sum -p 16 -c 0

需要注意这里的 -p 应该与每个节点上的卡数相同。

分批逐步测试连通性

HOSTS="172.19.110.211:8,172.19.52.211:8"

EXEC="all_reduce_test"
ARGS="-b 8K -e 4G -f 2 -d fp32 -o sum -p 8 -c 0"

START_N=16
END_N=400
STEP=8

for (( N=$START_N; N<=$END_N; N+=STEP )); do
    echo "Running with -n $N (using $((N/8)) pods)"
    mpirun -n $N -hosts $HOSTS $EXEC $ARGS
    sleep 10
done

采用二分发设置 START_N 和 END_N 可以快速找到机器中的异常点。

4. 单独运行 hccl-test 容器

nerdctl run --rm -it --ipc=host \
                     --device=/dev/davinci0 \
                     --device=/dev/davinci1 \
                     --device=/dev/davinci2 \
                     --device=/dev/davinci3 \
                     --device=/dev/davinci4 \
                     --device=/dev/davinci5 \
                     --device=/dev/davinci6 \
                     --device=/dev/davinci7 \
                     --device=/dev/davinci_manager \
                     --device=/dev/devmm_svm \
                     --device=/dev/hisi_hdc \
                     -v /usr/local/Ascend/driver:/usr/local/Ascend/driver \
                     -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ \
                     -v /usr/local/sbin/npu-smi:/usr/local/sbin/npu-smi \
                     -v /var/log/npu/conf/slog/slog.conf:/var/log/npu/conf/slog/slog.conf \
                     -v /var/log/npu/slog/:/var/log/npu/slog \
                     -v /var/log/npu/profiling/:/var/log/npu/profiling \
                     -v /var/log/npu/dump/:/var/log/npu/dump \
                     -v /var/log/npu/:/usr/slog \
                     shaowenchen/hccl-test:arm64-20240712v2 \
                     /bin/bash

推荐订阅源

陈少文的网站

1. 制作 hccl-test 镜像

2. 运行 Volcano Job

3. 运行 hccl-test 测试用例

4. 单独运行 hccl-test 容器