feat(rocm): Vendor SDK recipe and update to 10.1
Maintain the gfx1151 SDK locally using the official AMD archive. Use the TheRock patchelf pin and adapt LLVM auxiliary tools for pure Nix builds. Verify SDK installation, HIP and rocBLAS GPU operations, and rebuild its consumers. Document the internal rc3 stamp and upstream migration findings.
This commit is contained in:
@@ -52,11 +52,16 @@ let
|
||||
(old: {
|
||||
nativeBuildInputs = old.nativeBuildInputs ++ [ hipccFixed ];
|
||||
env.HIPFIRE_HIPCC = "${hipccFixed}/bin/hipcc";
|
||||
preBuild = (old.preBuild or "") + ''
|
||||
export CARGO_BUILD_JOBS="$NIX_BUILD_CORES"
|
||||
export HIPFIRE_PACK_JOBS="$NIX_BUILD_CORES"
|
||||
'';
|
||||
});
|
||||
|
||||
hipfireWithQuantize = pkgs.hipfire.overrideAttrs (_: {
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
export CARGO_BUILD_JOBS="$NIX_BUILD_CORES"
|
||||
cargo build --release -p hipfire-daemon
|
||||
cargo build --release -p hipfire-cli
|
||||
cargo build --release -p hipfire-quantize \
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <rocblas/rocblas.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
#define HIP_CHECK(call) do { \
|
||||
hipError_t error = (call); \
|
||||
if (error != hipSuccess) { \
|
||||
std::fprintf(stderr, "%s: %s\n", #call, hipGetErrorString(error)); \
|
||||
std::exit(1); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define BLAS_CHECK(call) do { \
|
||||
rocblas_status status = (call); \
|
||||
if (status != rocblas_status_success) { \
|
||||
std::fprintf(stderr, "%s: rocBLAS status %d\n", #call, int(status)); \
|
||||
std::exit(1); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
__global__ void vector_add(const float* a, const float* b, float* result) {
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
result[i] = a[i] + b[i];
|
||||
}
|
||||
|
||||
int main() {
|
||||
constexpr int count = 1024;
|
||||
float a[count], b[count], result[count];
|
||||
for (int i = 0; i < count; ++i) {
|
||||
a[i] = float(i);
|
||||
b[i] = float(count - i);
|
||||
}
|
||||
hipDeviceProp_t properties{};
|
||||
HIP_CHECK(hipGetDeviceProperties(&properties, 0));
|
||||
float *device_a, *device_b, *device_result;
|
||||
HIP_CHECK(hipMalloc(&device_a, sizeof(a)));
|
||||
HIP_CHECK(hipMalloc(&device_b, sizeof(b)));
|
||||
HIP_CHECK(hipMalloc(&device_result, sizeof(result)));
|
||||
HIP_CHECK(hipMemcpy(device_a, a, sizeof(a), hipMemcpyHostToDevice));
|
||||
HIP_CHECK(hipMemcpy(device_b, b, sizeof(b), hipMemcpyHostToDevice));
|
||||
vector_add<<<count / 256, 256>>>(device_a, device_b, device_result);
|
||||
HIP_CHECK(hipGetLastError());
|
||||
HIP_CHECK(hipMemcpy(result, device_result, sizeof(result), hipMemcpyDeviceToHost));
|
||||
for (int i = 0; i < count; ++i) {
|
||||
if (result[i] != float(count)) {
|
||||
std::fprintf(stderr, "vector add mismatch at %d: %f\n", i, result[i]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const float matrix_a[] = {1, 3, 2, 4};
|
||||
const float matrix_b[] = {5, 7, 6, 8};
|
||||
const float expected[] = {19, 43, 22, 50};
|
||||
float matrix_result[4];
|
||||
HIP_CHECK(hipMemcpy(device_a, matrix_a, sizeof(matrix_a), hipMemcpyHostToDevice));
|
||||
HIP_CHECK(hipMemcpy(device_b, matrix_b, sizeof(matrix_b), hipMemcpyHostToDevice));
|
||||
rocblas_handle handle;
|
||||
BLAS_CHECK(rocblas_create_handle(&handle));
|
||||
const float alpha = 1, beta = 0;
|
||||
BLAS_CHECK(rocblas_sgemm(handle, rocblas_operation_none, rocblas_operation_none,
|
||||
2, 2, 2, &alpha, device_a, 2, device_b, 2, &beta, device_result, 2));
|
||||
HIP_CHECK(hipMemcpy(matrix_result, device_result, sizeof(matrix_result), hipMemcpyDeviceToHost));
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
if (matrix_result[i] != expected[i]) {
|
||||
std::fprintf(stderr, "rocBLAS matrix multiply mismatch at %d: %f\n", i, matrix_result[i]);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
BLAS_CHECK(rocblas_destroy_handle(handle));
|
||||
HIP_CHECK(hipFree(device_a));
|
||||
HIP_CHECK(hipFree(device_b));
|
||||
HIP_CHECK(hipFree(device_result));
|
||||
std::printf("HIP vector add and rocBLAS matmul OK: %s (%s), %d elements\n",
|
||||
properties.name, properties.gcnArchName, count);
|
||||
}
|
||||
@@ -1,10 +1,8 @@
|
||||
{ pkgs, nix-strix-halo }:
|
||||
let
|
||||
source =
|
||||
(builtins.fromJSON (builtins.readFile (nix-strix-halo + "/pkgs/therock/sources/rocm.json")))
|
||||
.linux.gfx1151;
|
||||
in
|
||||
pkgs.callPackage (nix-strix-halo + "/pkgs/therock/rocm-sdk") {
|
||||
{ callPackage }:
|
||||
callPackage ./sdk.nix {
|
||||
patchelf = callPackage ./patchelf.nix { };
|
||||
target = "gfx1151";
|
||||
inherit (source) version url hash;
|
||||
version = "10.1.0";
|
||||
packageVersion = "10.1.0rc3";
|
||||
hash = "sha256-UN65kwNbwvalq6Xj3o1HbYxRKi7sLL/VqladceC4szc=";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
patchelf,
|
||||
stdenv,
|
||||
fetchFromGitHub,
|
||||
autoreconfHook,
|
||||
}:
|
||||
# Match TheRock's dockerfiles/build_manylinux_x86_64.Dockerfile. This includes
|
||||
# NixOS/patchelf#544 and accepts the SDK's librocsparse ELF when extending RPATH.
|
||||
(patchelf.override { inherit stdenv; }).overrideAttrs (old: {
|
||||
pname = "patchelf-therock";
|
||||
version = "0.18.0-therock-d0f70ee";
|
||||
src = fetchFromGitHub {
|
||||
owner = "NixOS";
|
||||
repo = "patchelf";
|
||||
rev = "d0f70eea5397606c486857e0a105e53ec123904a";
|
||||
hash = "sha256-CWAlYVtcswIpOZj5Qyxswsy1pAnPUTTUDapJO9ar0Mc=";
|
||||
};
|
||||
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ autoreconfHook ];
|
||||
doCheck = true;
|
||||
meta = old.meta // {
|
||||
changelog = "https://github.com/NixOS/patchelf/blob/d0f70eea5397606c486857e0a105e53ec123904a/ChangeLog.md";
|
||||
};
|
||||
})
|
||||
@@ -0,0 +1,232 @@
|
||||
# Adapted from hellas-ai/nix-strix-halo at b2969bd90fedc4a1f9d91648bfdf8f21479b8178,
|
||||
# pkgs/therock/rocm-sdk/default.nix. Maintained locally from ROCm 10.1 onward.
|
||||
{
|
||||
lib,
|
||||
stdenvNoCC,
|
||||
stdenv,
|
||||
binutils,
|
||||
jq,
|
||||
fetchurl,
|
||||
patchelf,
|
||||
libdrm,
|
||||
numactl,
|
||||
rdma-core,
|
||||
target,
|
||||
version,
|
||||
packageVersion ? version,
|
||||
url ? "https://stable.repo.amd.com/rocm/core/tarball/therock-dist-linux-${target}-${version}.tar.gz",
|
||||
hash,
|
||||
}:
|
||||
|
||||
stdenvNoCC.mkDerivation (finalAttrs: {
|
||||
pname = "therock-rocm-sdk-${target}";
|
||||
inherit version;
|
||||
|
||||
src = fetchurl {
|
||||
inherit url hash;
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
binutils
|
||||
jq
|
||||
patchelf
|
||||
];
|
||||
|
||||
propagatedBuildInputs = [
|
||||
libdrm
|
||||
numactl
|
||||
rdma-core
|
||||
];
|
||||
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
dontPatchELF = true;
|
||||
dontStrip = true;
|
||||
# TheRock binary SDK ships both lib and lib64 content. Nix's generic
|
||||
# move-lib64 hook cannot merge this split layout and leaves lib64 non-empty.
|
||||
dontMoveLib64 = true;
|
||||
|
||||
unpackPhase = ''
|
||||
runHook preUnpack
|
||||
tar -xzf "$src"
|
||||
runHook postUnpack
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
mkdir -p "$out"
|
||||
shopt -s dotglob nullglob
|
||||
|
||||
if [ -d install ]; then
|
||||
cp -R install/* "$out/"
|
||||
else
|
||||
entries=(*/)
|
||||
if [ "''${#entries[@]}" -eq 1 ] && [ -d "''${entries[0]}install" ]; then
|
||||
cp -R "''${entries[0]}install/"* "$out/"
|
||||
else
|
||||
cp -R ./* "$out/"
|
||||
fi
|
||||
fi
|
||||
|
||||
chmod -R u+w "$out"
|
||||
find "$out" -type f -name "*.so*" -exec chmod 755 {} \; 2>/dev/null || true
|
||||
find "$out/bin" "$out/llvm/bin" -type f -exec chmod 755 {} \; 2>/dev/null || true
|
||||
|
||||
# TheRock binary SDKs are built as relocatable Linux tarballs, not Nix
|
||||
# derivations. Make the shipped ELF binaries/libraries usable in pure Nix
|
||||
# build sandboxes so CMake probes such as FindHIP can execute hipconfig.
|
||||
gcc_runtime=${lib.escapeShellArg "${stdenv.cc.cc.lib}/lib"}
|
||||
libdrm_runtime=${lib.escapeShellArg "${lib.getLib libdrm}/lib"}
|
||||
numa_runtime=${lib.escapeShellArg "${lib.getLib numactl}/lib"}
|
||||
rdma_runtime=${lib.escapeShellArg "${lib.getLib rdma-core}/lib"}
|
||||
dynamic_linker=${lib.escapeShellArg stdenv.cc.bintools.dynamicLinker}
|
||||
runtime_paths=(
|
||||
"$gcc_runtime"
|
||||
"$libdrm_runtime"
|
||||
"$numa_runtime"
|
||||
"$rdma_runtime"
|
||||
)
|
||||
|
||||
patch_elf() {
|
||||
local elf="$1"
|
||||
|
||||
# Use TheRock's pinned patchelf with NixOS/patchelf#544: late PHDR
|
||||
# tables in kpack split ELFs can be rewritten without corruption.
|
||||
if old_rpath=$(patchelf --print-rpath "$elf" 2>/dev/null); then
|
||||
new_rpath="$old_rpath"
|
||||
for runtime_path in "''${runtime_paths[@]}"; do
|
||||
case ":$new_rpath:" in
|
||||
*":$runtime_path:"*) ;;
|
||||
*) new_rpath="''${new_rpath:+$new_rpath:}$runtime_path" ;;
|
||||
esac
|
||||
done
|
||||
patchelf --set-rpath "$new_rpath" "$elf" || {
|
||||
echo "failed to patch RPATH: $elf" >&2
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
|
||||
if patchelf --print-interpreter "$elf" >/dev/null 2>&1; then
|
||||
patchelf --set-interpreter "$dynamic_linker" "$elf"
|
||||
fi
|
||||
}
|
||||
|
||||
# LLVM 24 also invokes auxiliary tools such as llvm-offload-binary.
|
||||
# Patch every top-level ELF tool, including these compiler subprocesses.
|
||||
for dir in "$out/bin" "$out/llvm/bin" "$out/lib/llvm/bin"; do
|
||||
[ -d "$dir" ] || continue
|
||||
for tool in "$dir"/*; do
|
||||
[ -f "$tool" ] || continue
|
||||
patch_elf "$tool"
|
||||
done
|
||||
done
|
||||
|
||||
for dir in \
|
||||
"$out/lib" \
|
||||
"$out/lib/rocm_sysdeps/lib" \
|
||||
"$out/lib/host-math/lib" \
|
||||
"$out/llvm/lib" \
|
||||
"$out/lib/llvm/lib" \
|
||||
"$out/share/amd_smi/amdsmi"; do
|
||||
[ -d "$dir" ] || continue
|
||||
while IFS= read -r -d "" elf; do
|
||||
patch_elf "$elf"
|
||||
done < <(find "$dir" -maxdepth 1 -type f \( -name "*.so" -o -name "*.so.*" \) -print0)
|
||||
done
|
||||
|
||||
# GCC's Nix wrapper enables the x86 TLS dialect globally. Clang's HIP
|
||||
# driver forwards that option to amdgcn too, where it is invalid.
|
||||
cc_cflags_before="$(sed -E 's/(^|[[:space:]])-mtls-dialect=[^[:space:]]+//g' ${stdenv.cc}/nix-support/cc-cflags-before)"
|
||||
cc_cflags="$(cat ${stdenv.cc}/nix-support/cc-cflags)"
|
||||
libc_cflags="$(cat ${stdenv.cc}/nix-support/libc-cflags)"
|
||||
libc_crt1_cflags="$(cat ${stdenv.cc}/nix-support/libc-crt1-cflags)"
|
||||
libc_ldflags="$(cat ${stdenv.cc}/nix-support/libc-ldflags)"
|
||||
cc_ldflags="$(cat ${stdenv.cc}/nix-support/cc-ldflags)"
|
||||
libc="$(cat ${stdenv.cc}/nix-support/orig-libc)"
|
||||
dynamic_linker="$(cat ${stdenv.cc}/nix-support/dynamic-linker)"
|
||||
|
||||
cat > "$out/bin/therock-hip-clang++" <<EOF
|
||||
#!/bin/sh
|
||||
exec "$out/lib/llvm/bin/clang++" \\
|
||||
--gcc-toolchain=${stdenv.cc.cc} \\
|
||||
--rocm-path="$out" \\
|
||||
$cc_cflags_before \\
|
||||
$cc_cflags \\
|
||||
$libc_cflags \\
|
||||
$libc_crt1_cflags \\
|
||||
"\$@" \\
|
||||
-L$libc/lib \\
|
||||
-Wl,--dynamic-linker=$dynamic_linker \\
|
||||
-Wl,-rpath,${stdenv.cc.cc.lib}/lib \\
|
||||
-Wl,-rpath,$libc/lib \\
|
||||
$libc_ldflags \\
|
||||
$cc_ldflags
|
||||
EOF
|
||||
sed -i 's/^ //' "$out/bin/therock-hip-clang++"
|
||||
chmod 755 "$out/bin/therock-hip-clang++"
|
||||
|
||||
# TheRock's CMake HIP helper normally re-enters hipcc, which then invokes
|
||||
# the raw bundled clang++ during link steps. That loses Nix's libc/libstdc++
|
||||
# search paths and fails to link extension modules in pure builds. Keep the
|
||||
# CMake contract (first argv is HIP_CLANG_PATH), but route the actual link
|
||||
# through the Nix-aware compiler wrapper above.
|
||||
cat > "$out/bin/hipcc_cmake_linker_helper" <<EOF
|
||||
#!/bin/sh
|
||||
[ "\$#" -gt 0 ] && shift
|
||||
exec "$out/bin/therock-hip-clang++" "\$@"
|
||||
EOF
|
||||
sed -i 's/^ //' "$out/bin/hipcc_cmake_linker_helper"
|
||||
chmod 755 "$out/bin/hipcc_cmake_linker_helper"
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
doInstallCheck = true;
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
jq -e --arg version '${version}' --arg packageVersion '${packageVersion}' \
|
||||
'.rocm_version == $version and .rocm_package_version == $packageVersion' \
|
||||
$out/share/therock/therock_manifest.json
|
||||
$out/bin/hipconfig --version
|
||||
$out/lib/llvm/bin/clang++ --version
|
||||
$out/lib/llvm/bin/llvm-offload-binary --version
|
||||
$out/lib/llvm/bin/llvm-readobj --file-headers $out/lib/libamdhip64.so
|
||||
readelf --program-headers $out/lib/libhsa-runtime64.so.1 > program-headers.log 2>&1
|
||||
if grep -E 'Error:|not covered by a LOAD segment' program-headers.log; then
|
||||
exit 1
|
||||
fi
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit target patchelf packageVersion;
|
||||
tests.hip = stdenv.mkDerivation {
|
||||
pname = "therock-hip-smoke-${target}";
|
||||
inherit version;
|
||||
dontUnpack = true;
|
||||
buildPhase = ''
|
||||
${finalAttrs.finalPackage}/bin/therock-hip-clang++ \
|
||||
-x hip --offload-arch=${target} ${./hip-smoke.cpp} \
|
||||
-L${finalAttrs.finalPackage}/lib -Wl,-rpath,${finalAttrs.finalPackage}/lib \
|
||||
-lrocblas -o hip-smoke
|
||||
'';
|
||||
installPhase = ''
|
||||
install -Dm755 hip-smoke $out/bin/hip-smoke
|
||||
'';
|
||||
meta = {
|
||||
description = "HIP vector-add and rocBLAS GPU checks for the packaged TheRock SDK";
|
||||
platforms = [ "x86_64-linux" ];
|
||||
mainProgram = "hip-smoke";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
meta = {
|
||||
description = "TheRock ROCm SDK binary tarball for ${target}";
|
||||
homepage = "https://github.com/ROCm/TheRock";
|
||||
license = lib.licenses.mit;
|
||||
sourceProvenance = [ lib.sourceTypes.binaryNativeCode ];
|
||||
platforms = [ "x86_64-linux" ];
|
||||
};
|
||||
})
|
||||
Reference in New Issue
Block a user