Nobody expects the Red Team

Too many changes to list, but broadly:
* Remove Intel GPU support from the compiler
* Add AMD GPU support to the compiler
* Remove Intel GPU host code
* Add AMD GPU host code
* More device instructions. From 40 to 68
* More host functions. From 48 to 184
* Add proof of concept implementation of OptiX framework
* Add minimal support of cuDNN, cuBLAS, cuSPARSE, cuFFT, NCCL, NVML
* Improve ZLUDA launcher for Windows

Author: vosen

.cargo/config.toml

@@ -1,2 +1,8 @@
-[target."x86_64-pc-windows-gnu"]
-rustflags = ["-C", "link-self-contained=y"]
+[target."x86_64-unknown-linux-gnu"]
+rustflags = ["-C", "target-cpu=x86-64-v2"]
+
+[target."x86_64-pc-windows-msvc"]
+rustflags = ["-C", "target-cpu=x86-64-v2"]
+
+[alias]
+xtask = "run --package xtask --"

.devcontainer/Dockerfile

@@ -0,0 +1,6 @@
+# syntax = devthefuture/dockerfile-x
+
+# This duplicate FROM is here purely to make dev containers happy,
+# Otherwise it tries to parse the file (whyyy???) and chokes on custom syntax
+FROM ubuntu:22.04
+INCLUDE ./Dockerfile-common

.devcontainer/Dockerfile-common

@@ -0,0 +1,64 @@
+WORKDIR /root
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  ca-certificates \
+  nano \
+  wget \
+  curl \
+  gnupg \
+  ripgrep \
+  ltrace \
+  file\
+  python3-minimal \
+  build-essential \
+  git \
+  cmake \
+  ninja-build
+ENV PATH="${PATH}:/opt/rocm/bin:/opt/rocm/llvm/bin:/usr/local/cuda/bin/"
+
+
+ARG CUDA_VERSION=11-8
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb && \
+  dpkg -i cuda-keyring_1.0-1_all.deb && \
+  apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  nvidia-headless-no-dkms-515 \
+  nvidia-utils-515 \
+  cuda-cudart-${CUDA_VERSION} \
+  cuda-compiler-${CUDA_VERSION} \
+  libcufft-dev-${CUDA_VERSION} \
+  libcusparse-dev-${CUDA_VERSION} \
+  libcublas-dev-${CUDA_VERSION} \
+  cuda-nvml-dev-${CUDA_VERSION} \
+  libcudnn8-dev
+
+ARG RUST_VERSION=1.66.1
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=${RUST_VERSION}
+RUN . $HOME/.cargo/env && cargo install bindgen-cli --locked
+
+ARG ROCM_VERSION=5.7.3
+RUN echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" > /etc/apt/preferences.d/rocm-pin-600
+RUN mkdir --parents --mode=0755 /etc/apt/keyrings && \
+  sh -c 'wget https://repo.radeon.com/rocm/rocm.gpg.key -O - |  gpg --dearmor | tee /etc/apt/keyrings/rocm.gpg > /dev/null' && \
+  sh -c 'echo deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} jammy main > /etc/apt/sources.list.d/rocm.list' && \
+  apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  rocminfo \
+  rocm-gdb \
+  rocprofiler \
+  rocm-smi-lib \
+  hip-runtime-amd \
+  comgr \
+  hipblaslt-dev \
+  hipfft-dev \
+  rocblas-dev \
+  rocsolver-dev \
+  rocsparse-dev \
+  miopen-hip-dev \
+  rocm-device-libs && \
+  echo 'export PATH="$PATH:/opt/rocm/bin"' > /etc/profile.d/rocm.sh && \
+  echo '/opt/rocm/lib' > /etc/ld.so.conf.d/rocm.conf && \
+  ldconfig
+
+# Default to a login shell
+CMD ["bash", "-l"]

.devcontainer/Dockerfile-el8_8

@@ -0,0 +1,72 @@
+FROM rockylinux:8.8
+
+WORKDIR /root
+
+RUN dnf -y --setopt=install_weak_deps=False install \
+    nano \
+    wget \
+    curl \
+    ltrace \
+    file \
+    python3 \
+    git \
+    gcc \
+    gcc-c++ \
+    cmake
+
+RUN wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+    rpm -ivh epel-release-latest-8.noarch.rpm && \
+    dnf -y --setopt=install_weak_deps=False install 'dnf-command(config-manager)' && \
+    crb enable && \
+    dnf -y --setopt=install_weak_deps=False install \
+    ripgrep \
+    ninja-build
+
+ARG CUDA_VERSION=11-8
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo && \
+    dnf -y --setopt=install_weak_deps=False module install \
+    nvidia-driver:515 && \
+    dnf -y --setopt=install_weak_deps=False install \
+    cuda-cudart-${CUDA_VERSION} \
+    cuda-compiler-${CUDA_VERSION} \
+    libcufft-devel-${CUDA_VERSION} \
+    libcusparse-devel-${CUDA_VERSION} \
+    libcublas-devel-${CUDA_VERSION} \
+    cuda-nvml-devel-${CUDA_VERSION} \
+    libcudnn8-devel
+
+ARG RUST_VERSION=1.66.1
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=${RUST_VERSION}
+RUN . $HOME/.cargo/env && cargo install bindgen-cli --locked
+
+ARG ROCM_VERSION=5.7.1
+RUN sh -c 'echo -e "[ROCm-${ROCM_VERSION}]\n\
+name=ROCm${ROCM_VERSION}\n\
+baseurl=https://repo.radeon.com/rocm/rhel8/${ROCM_VERSION}/main\n\
+enabled=1\n\
+priority=50\n\
+gpgcheck=1\n\
+gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key"' \
+    > /etc/yum.repos.d/rocm.repo && \
+    dnf -y --setopt=install_weak_deps=False install \
+    rocminfo \
+    rocm-gdb \
+    rocprofiler \
+    rocm-smi-lib \
+    hip-runtime-amd \
+    comgr \
+    hipblaslt-devel \
+    hipfft-devel \
+    rocblas-devel \
+    rocsolver-devel \
+    rocsparse-devel \
+    miopen-hip-devel \
+    rocm-device-libs && \
+    echo 'export PATH="$PATH:/opt/rocm/bin"' > /etc/profile.d/rocm.sh && \
+    echo '/opt/rocm/lib' > /etc/ld.so.conf.d/rocm.conf && \
+    ldconfig
+
+# Default to a login shell
+CMD ["bash", "-l"]

.devcontainer/Dockerfile-xgboost

@@ -0,0 +1,16 @@
+# syntax = devthefuture/dockerfile-x
+FROM ubuntu:22.04
+INCLUDE ./Dockerfile-common
+
+ARG XGBOOST_VERSION=2.0.3
+RUN git clone --branch "v${XGBOOST_VERSION}" --recurse-submodules https://github.com/dmlc/xgboost.git && \
+    cd xgboost && \
+    # Broken test, segfaults on normal CUDA
+    sed -i 's/TEST(Allocator, OOM) {/TEST(Allocator, OOM) { GTEST_SKIP();/g' tests/cpp/common/test_device_helpers.cu && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DUSE_CUDA=ON -GNinja && \
+    ninja
+
+#
+

.devcontainer/devcontainer.json

@@ -0,0 +1,39 @@
+// For format details, see https://aka.ms/devcontainer.json
+{
+	"name": "ZLUDA",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"securityOpt": [
+		"seccomp=unconfined"
+	],
+	// Make NVIDIA and AMD GPUs available
+	"runArgs": [
+		// Uncomment on newer docker/podman
+		//"--runtime=nvidia",
+		"--device=/dev/kfd",
+		"--device=/dev/dri",
+		"--group-add=video"
+	],
+	// Cache cargo packages and compiled ZLUDA kernels
+	"initializeCommand": "mkdir -p ${localEnv:HOME}/.cargo/git ${localEnv:HOME}/.cargo/registry ${localEnv:HOME}/.cache/ZLUDA",
+	"mounts": [
+		{
+			"source": "${localEnv:HOME}/.cargo/git",
+			"target": "/root/.cargo/git",
+			"type": "bind"
+		},
+		{
+			"source": "${localEnv:HOME}/.cargo/registry",
+			"target": "/root/.cargo/registry",
+			"type": "bind"
+		},
+		{
+			"source": "${localEnv:HOME}/.cache/ZLUDA",
+			"target": "/root/.cache/ZLUDA",
+			"type": "bind"
+		}
+	],
+	// Rootless docker requires logging as root: https://aka.ms/dev-containers-non-root.
+	"remoteUser": "root"
+}

.gitattributes

@@ -0,0 +1,3 @@
+ext/** linguist-vendored
+atiadlxx-sys/include/* linguist-vendored
+*.ptx linguist-language=Assembly

.github/workflows/rust.yml

@@ -1,7 +1,5 @@
-[submodule "ext/spirv-tools"]
-	path = ext/spirv-tools
-	url = https://github.com/KhronosGroup/SPIRV-Tools
-	branch = master
-[submodule "ext/spirv-headers"]
-	path = ext/spirv-headers
-	url = https://github.com/KhronosGroup/SPIRV-Headers
+[submodule "ext/llvm-project"]
+	path = ext/llvm-project
+	url = https://github.com/llvm/llvm-project.git
+	branch = release/15.x
+	shallow = true