16166f115b
Maintain the gfx1151 SDK locally using the official AMD archive. Use the TheRock patchelf pin and adapt LLVM auxiliary tools for pure Nix builds. Verify SDK installation, HIP and rocBLAS GPU operations, and rebuild its consumers. Document the internal rc3 stamp and upstream migration findings.
233 lines
7.8 KiB
Nix
233 lines
7.8 KiB
Nix
# Adapted from hellas-ai/nix-strix-halo at b2969bd90fedc4a1f9d91648bfdf8f21479b8178,
|
|
# pkgs/therock/rocm-sdk/default.nix. Maintained locally from ROCm 10.1 onward.
|
|
{
|
|
lib,
|
|
stdenvNoCC,
|
|
stdenv,
|
|
binutils,
|
|
jq,
|
|
fetchurl,
|
|
patchelf,
|
|
libdrm,
|
|
numactl,
|
|
rdma-core,
|
|
target,
|
|
version,
|
|
packageVersion ? version,
|
|
url ? "https://stable.repo.amd.com/rocm/core/tarball/therock-dist-linux-${target}-${version}.tar.gz",
|
|
hash,
|
|
}:
|
|
|
|
stdenvNoCC.mkDerivation (finalAttrs: {
|
|
pname = "therock-rocm-sdk-${target}";
|
|
inherit version;
|
|
|
|
src = fetchurl {
|
|
inherit url hash;
|
|
};
|
|
|
|
nativeBuildInputs = [
|
|
binutils
|
|
jq
|
|
patchelf
|
|
];
|
|
|
|
propagatedBuildInputs = [
|
|
libdrm
|
|
numactl
|
|
rdma-core
|
|
];
|
|
|
|
dontConfigure = true;
|
|
dontBuild = true;
|
|
dontPatchELF = true;
|
|
dontStrip = true;
|
|
# TheRock binary SDK ships both lib and lib64 content. Nix's generic
|
|
# move-lib64 hook cannot merge this split layout and leaves lib64 non-empty.
|
|
dontMoveLib64 = true;
|
|
|
|
unpackPhase = ''
|
|
runHook preUnpack
|
|
tar -xzf "$src"
|
|
runHook postUnpack
|
|
'';
|
|
|
|
installPhase = ''
|
|
runHook preInstall
|
|
|
|
mkdir -p "$out"
|
|
shopt -s dotglob nullglob
|
|
|
|
if [ -d install ]; then
|
|
cp -R install/* "$out/"
|
|
else
|
|
entries=(*/)
|
|
if [ "''${#entries[@]}" -eq 1 ] && [ -d "''${entries[0]}install" ]; then
|
|
cp -R "''${entries[0]}install/"* "$out/"
|
|
else
|
|
cp -R ./* "$out/"
|
|
fi
|
|
fi
|
|
|
|
chmod -R u+w "$out"
|
|
find "$out" -type f -name "*.so*" -exec chmod 755 {} \; 2>/dev/null || true
|
|
find "$out/bin" "$out/llvm/bin" -type f -exec chmod 755 {} \; 2>/dev/null || true
|
|
|
|
# TheRock binary SDKs are built as relocatable Linux tarballs, not Nix
|
|
# derivations. Make the shipped ELF binaries/libraries usable in pure Nix
|
|
# build sandboxes so CMake probes such as FindHIP can execute hipconfig.
|
|
gcc_runtime=${lib.escapeShellArg "${stdenv.cc.cc.lib}/lib"}
|
|
libdrm_runtime=${lib.escapeShellArg "${lib.getLib libdrm}/lib"}
|
|
numa_runtime=${lib.escapeShellArg "${lib.getLib numactl}/lib"}
|
|
rdma_runtime=${lib.escapeShellArg "${lib.getLib rdma-core}/lib"}
|
|
dynamic_linker=${lib.escapeShellArg stdenv.cc.bintools.dynamicLinker}
|
|
runtime_paths=(
|
|
"$gcc_runtime"
|
|
"$libdrm_runtime"
|
|
"$numa_runtime"
|
|
"$rdma_runtime"
|
|
)
|
|
|
|
patch_elf() {
|
|
local elf="$1"
|
|
|
|
# Use TheRock's pinned patchelf with NixOS/patchelf#544: late PHDR
|
|
# tables in kpack split ELFs can be rewritten without corruption.
|
|
if old_rpath=$(patchelf --print-rpath "$elf" 2>/dev/null); then
|
|
new_rpath="$old_rpath"
|
|
for runtime_path in "''${runtime_paths[@]}"; do
|
|
case ":$new_rpath:" in
|
|
*":$runtime_path:"*) ;;
|
|
*) new_rpath="''${new_rpath:+$new_rpath:}$runtime_path" ;;
|
|
esac
|
|
done
|
|
patchelf --set-rpath "$new_rpath" "$elf" || {
|
|
echo "failed to patch RPATH: $elf" >&2
|
|
return 1
|
|
}
|
|
fi
|
|
|
|
if patchelf --print-interpreter "$elf" >/dev/null 2>&1; then
|
|
patchelf --set-interpreter "$dynamic_linker" "$elf"
|
|
fi
|
|
}
|
|
|
|
# LLVM 24 also invokes auxiliary tools such as llvm-offload-binary.
|
|
# Patch every top-level ELF tool, including these compiler subprocesses.
|
|
for dir in "$out/bin" "$out/llvm/bin" "$out/lib/llvm/bin"; do
|
|
[ -d "$dir" ] || continue
|
|
for tool in "$dir"/*; do
|
|
[ -f "$tool" ] || continue
|
|
patch_elf "$tool"
|
|
done
|
|
done
|
|
|
|
for dir in \
|
|
"$out/lib" \
|
|
"$out/lib/rocm_sysdeps/lib" \
|
|
"$out/lib/host-math/lib" \
|
|
"$out/llvm/lib" \
|
|
"$out/lib/llvm/lib" \
|
|
"$out/share/amd_smi/amdsmi"; do
|
|
[ -d "$dir" ] || continue
|
|
while IFS= read -r -d "" elf; do
|
|
patch_elf "$elf"
|
|
done < <(find "$dir" -maxdepth 1 -type f \( -name "*.so" -o -name "*.so.*" \) -print0)
|
|
done
|
|
|
|
# GCC's Nix wrapper enables the x86 TLS dialect globally. Clang's HIP
|
|
# driver forwards that option to amdgcn too, where it is invalid.
|
|
cc_cflags_before="$(sed -E 's/(^|[[:space:]])-mtls-dialect=[^[:space:]]+//g' ${stdenv.cc}/nix-support/cc-cflags-before)"
|
|
cc_cflags="$(cat ${stdenv.cc}/nix-support/cc-cflags)"
|
|
libc_cflags="$(cat ${stdenv.cc}/nix-support/libc-cflags)"
|
|
libc_crt1_cflags="$(cat ${stdenv.cc}/nix-support/libc-crt1-cflags)"
|
|
libc_ldflags="$(cat ${stdenv.cc}/nix-support/libc-ldflags)"
|
|
cc_ldflags="$(cat ${stdenv.cc}/nix-support/cc-ldflags)"
|
|
libc="$(cat ${stdenv.cc}/nix-support/orig-libc)"
|
|
dynamic_linker="$(cat ${stdenv.cc}/nix-support/dynamic-linker)"
|
|
|
|
cat > "$out/bin/therock-hip-clang++" <<EOF
|
|
#!/bin/sh
|
|
exec "$out/lib/llvm/bin/clang++" \\
|
|
--gcc-toolchain=${stdenv.cc.cc} \\
|
|
--rocm-path="$out" \\
|
|
$cc_cflags_before \\
|
|
$cc_cflags \\
|
|
$libc_cflags \\
|
|
$libc_crt1_cflags \\
|
|
"\$@" \\
|
|
-L$libc/lib \\
|
|
-Wl,--dynamic-linker=$dynamic_linker \\
|
|
-Wl,-rpath,${stdenv.cc.cc.lib}/lib \\
|
|
-Wl,-rpath,$libc/lib \\
|
|
$libc_ldflags \\
|
|
$cc_ldflags
|
|
EOF
|
|
sed -i 's/^ //' "$out/bin/therock-hip-clang++"
|
|
chmod 755 "$out/bin/therock-hip-clang++"
|
|
|
|
# TheRock's CMake HIP helper normally re-enters hipcc, which then invokes
|
|
# the raw bundled clang++ during link steps. That loses Nix's libc/libstdc++
|
|
# search paths and fails to link extension modules in pure builds. Keep the
|
|
# CMake contract (first argv is HIP_CLANG_PATH), but route the actual link
|
|
# through the Nix-aware compiler wrapper above.
|
|
cat > "$out/bin/hipcc_cmake_linker_helper" <<EOF
|
|
#!/bin/sh
|
|
[ "\$#" -gt 0 ] && shift
|
|
exec "$out/bin/therock-hip-clang++" "\$@"
|
|
EOF
|
|
sed -i 's/^ //' "$out/bin/hipcc_cmake_linker_helper"
|
|
chmod 755 "$out/bin/hipcc_cmake_linker_helper"
|
|
|
|
runHook postInstall
|
|
'';
|
|
|
|
doInstallCheck = true;
|
|
installCheckPhase = ''
|
|
runHook preInstallCheck
|
|
jq -e --arg version '${version}' --arg packageVersion '${packageVersion}' \
|
|
'.rocm_version == $version and .rocm_package_version == $packageVersion' \
|
|
$out/share/therock/therock_manifest.json
|
|
$out/bin/hipconfig --version
|
|
$out/lib/llvm/bin/clang++ --version
|
|
$out/lib/llvm/bin/llvm-offload-binary --version
|
|
$out/lib/llvm/bin/llvm-readobj --file-headers $out/lib/libamdhip64.so
|
|
readelf --program-headers $out/lib/libhsa-runtime64.so.1 > program-headers.log 2>&1
|
|
if grep -E 'Error:|not covered by a LOAD segment' program-headers.log; then
|
|
exit 1
|
|
fi
|
|
runHook postInstallCheck
|
|
'';
|
|
|
|
passthru = {
|
|
inherit target patchelf packageVersion;
|
|
tests.hip = stdenv.mkDerivation {
|
|
pname = "therock-hip-smoke-${target}";
|
|
inherit version;
|
|
dontUnpack = true;
|
|
buildPhase = ''
|
|
${finalAttrs.finalPackage}/bin/therock-hip-clang++ \
|
|
-x hip --offload-arch=${target} ${./hip-smoke.cpp} \
|
|
-L${finalAttrs.finalPackage}/lib -Wl,-rpath,${finalAttrs.finalPackage}/lib \
|
|
-lrocblas -o hip-smoke
|
|
'';
|
|
installPhase = ''
|
|
install -Dm755 hip-smoke $out/bin/hip-smoke
|
|
'';
|
|
meta = {
|
|
description = "HIP vector-add and rocBLAS GPU checks for the packaged TheRock SDK";
|
|
platforms = [ "x86_64-linux" ];
|
|
mainProgram = "hip-smoke";
|
|
};
|
|
};
|
|
};
|
|
|
|
meta = {
|
|
description = "TheRock ROCm SDK binary tarball for ${target}";
|
|
homepage = "https://github.com/ROCm/TheRock";
|
|
license = lib.licenses.mit;
|
|
sourceProvenance = [ lib.sourceTypes.binaryNativeCode ];
|
|
platforms = [ "x86_64-linux" ];
|
|
};
|
|
})
|