[docker/arm64] Build docker images (with suffix) for arm64 (#31861)

This PR adds Docker image builds for arm64 (aarch64). The images will be tagged with a suffix `-aarch64`. In a future retagging step, we can combine the existing x86_64 images with the aarch64 images for true multiplatform images.

Changes needed to make this work:
- Move libgcc to libgcc-ng because the former does not exist on conda for aarch64
- Upgrade Python 3.7.7 to 3.7.16 because the former does not exist on conda for aarch74

Please also note that aarch64 base images only exist for CUDA >= 11.2.

Lastly, we are not building ray-ml docker images. This is because not all current dependencies are compatible with aarch64 - specifically, tensorflow requires at least 1.10.1 and torch 1.13.0 does not seem to be released for linux aarch64.

Signed-off-by: Kai Fricke <kai@anyscale.com>

.buildkite/pipeline.arm64.yml

@@ -54,3 +54,83 @@
 #    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
 #    # Upload to latest directory.
 #    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+
+
+- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py37 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py37 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py38 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py38 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py39 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py39 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (1/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cpu cu112 --build-type BUILDKITE --build-base --suffix aarch64
+
+- label: ":mechanical_arm: :docker: Build Images: py310 [aarch64] (2/2)"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - pip install -q docker aws_requests_auth boto3
+    - if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then python .buildkite/copy_files.py --destination docker_login; fi
+    - python ./ci/build/build-docker-images.py --py-versions py310 --device-types cu113 cu116 --build-type BUILDKITE --build-base --suffix aarch64

build-docker.sh

@@ -8,7 +8,7 @@ set -x
 GPU=""
 BASE_IMAGE="ubuntu:focal"
 WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
-PYTHON_VERSION="3.7.7"
+PYTHON_VERSION="3.7.16"
 
 
 while [[ $# -gt 0 ]]