Author: Joseph Huber Date: 2026-11-01T15:30:24-06:00 New Revision: eb0f3cb8addd6cf1ccd8e55101f528b52fd360ae
URL: https://github.com/llvm/llvm-project/commit/eb0f3cb8addd6cf1ccd8e55101f528b52fd360ae DIFF: https://github.com/llvm/llvm-project/commit/eb0f3cb8addd6cf1ccd8e55101f528b52fd360ae.diff LOG: [Clang] Support `--offload-compress` for offload binaries (#227765) Summary: https://github.com/llvm/llvm-project/pull/222774 added support for compressing binaries. This PR does the minimal work required to hook this up to the existing handling. The end goal is to be able to replace the bundler in HIP. The one change is moving the `--verbose` handling from the shared helper so this can be unified. Added: clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-compressed.c llvm/test/tools/llvm-offload-binary/compress-level-zlib.test Modified: clang/include/clang/Driver/CommonArgs.h clang/lib/Driver/ToolChains/Clang.cpp clang/lib/Driver/ToolChains/CommonArgs.cpp clang/lib/Driver/ToolChains/HIPUtility.cpp clang/test/Driver/hip-offload-compress-zstd.hip llvm/docs/CommandGuide/llvm-offload-binary.md llvm/test/tools/llvm-offload-binary/compress.test llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp Removed: ################################################################################ diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h index e357dbf20e12f6..138e18f8641b90 100644 --- a/clang/include/clang/Driver/CommonArgs.h +++ b/clang/include/clang/Driver/CommonArgs.h @@ -263,7 +263,8 @@ void addOutlineAtomicsArgs(const Driver &D, const ToolChain &TC, llvm::opt::ArgStringList &CmdArgs, const llvm::Triple &Triple); void addOffloadCompressArgs(const llvm::opt::ArgList &TCArgs, - llvm::opt::ArgStringList &CmdArgs); + llvm::opt::ArgStringList &CmdArgs, + bool EmitCompressionFormat = false); void addMCModel(const Driver &D, const llvm::opt::ArgList &Args, const llvm::Triple &Triple, const llvm::Reloc::Model &RelocationModel, diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index 1bce61d2cfbeaa..61a81f33ab22dd 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -9705,6 +9705,8 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(TCArgs.MakeArgString(UB)); } addOffloadCompressArgs(TCArgs, CmdArgs); + if (TCArgs.hasArg(options::OPT_v)) + CmdArgs.push_back("--verbose"); // All the inputs are encoded as commands. C.addCommand(std::make_unique<Command>( JA, *this, ResponseFileSupport::None(), @@ -9762,6 +9764,11 @@ void OffloadPackager::ConstructJob(Compilation &C, const JobAction &JA, CmdArgs.push_back(Args.MakeArgString("--image=" + llvm::join(Parts, ","))); } + // FIXME: --offload-compress is HIP only until the OpenMP runtime accepts it. + if (JA.isDeviceOffloading(Action::OFK_HIP) && + Args.hasFlag(options::OPT_fgpu_rdc, options::OPT_fno_gpu_rdc, false)) + addOffloadCompressArgs(Args, CmdArgs, /*EmitCompressionFormat=*/true); + C.addCommand(std::make_unique<Command>( JA, *this, ResponseFileSupport::AtFileUTF8(), Args.MakeArgString(getToolChain().GetProgramPath(getShortName())), diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 7400d0f4b338eb..d7ce5270243442 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -3034,15 +3034,24 @@ void tools::addOutlineAtomicsArgs(const Driver &D, const ToolChain &TC, } void tools::addOffloadCompressArgs(const llvm::opt::ArgList &TCArgs, - llvm::opt::ArgStringList &CmdArgs) { + llvm::opt::ArgStringList &CmdArgs, + bool EmitCompressionFormat) { if (TCArgs.hasFlag(options::OPT_offload_compress, options::OPT_no_offload_compress, false)) CmdArgs.push_back("--compress"); - if (TCArgs.hasArg(options::OPT_v)) - CmdArgs.push_back("--verbose"); - if (auto *Arg = TCArgs.getLastArg(options::OPT_offload_compression_level_EQ)) + if (auto *Arg = + TCArgs.getLastArg(options::OPT_offload_compression_level_EQ)) { + // llvm-offload-binary rejects a level unless the format is named. Pick the + // same default the tool would: zstd, or zlib when zstd is not built. + if (EmitCompressionFormat) { + const char *Format = + llvm::compression::zstd::isAvailable() ? "zstd" : "zlib"; + CmdArgs.push_back( + TCArgs.MakeArgString(Twine("--compression-format=") + Format)); + } CmdArgs.push_back( TCArgs.MakeArgString(Twine("--compression-level=") + Arg->getValue())); + } } void tools::addMCModel(const Driver &D, const llvm::opt::ArgList &Args, diff --git a/clang/lib/Driver/ToolChains/HIPUtility.cpp b/clang/lib/Driver/ToolChains/HIPUtility.cpp index 9c4a67b2f182c6..e19cfa0349bfe6 100644 --- a/clang/lib/Driver/ToolChains/HIPUtility.cpp +++ b/clang/lib/Driver/ToolChains/HIPUtility.cpp @@ -82,6 +82,8 @@ void HIP::constructHIPFatbinCommand(Compilation &C, const JobAction &JA, BundlerArgs.push_back(BundlerOutputArg); addOffloadCompressArgs(Args, BundlerArgs); + if (Args.hasArg(options::OPT_v)) + BundlerArgs.push_back("--verbose"); const char *Bundler = Args.MakeArgString( T.getToolChain().GetProgramPath("clang-offload-bundler")); diff --git a/clang/test/Driver/hip-offload-compress-zstd.hip b/clang/test/Driver/hip-offload-compress-zstd.hip index a231197af44f83..c8899a1b97e720 100644 --- a/clang/test/Driver/hip-offload-compress-zstd.hip +++ b/clang/test/Driver/hip-offload-compress-zstd.hip @@ -12,4 +12,32 @@ // RUN: -o %t.bc \ // RUN: 2>&1 | FileCheck %s --check-prefix=NEWDRIVER -// NEWDRIVER: clang-linker-wrapper{{.*}}"--compress" "--verbose" "--compression-level=9" +// NEWDRIVER: llvm-offload-binary{{.*}}"--image={{[^"]*}}arch=gfx1101{{[^"]*}}" "--compress" "--compression-format=zstd" "--compression-level=9"{{$}} +// NEWDRIVER: clang-linker-wrapper{{.*}}"--compress" "--compression-level=9" +// NEWDRIVER-NOT: "--verbose" + +// RUN: %clang -### --target=x86_64-linux-gnu \ +// RUN: -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \ +// RUN: -nogpuinc -nogpulib -c %S/Inputs/hip_multiple_inputs/a.cu \ +// RUN: --offload-compress \ +// RUN: 2>&1 | FileCheck %s --check-prefix=NORDC + +// NORDC: llvm-offload-binary{{.*}}"--image={{[^"]*}}arch=gfx1101{{[^"]*}}"{{$}} +// NORDC-NOT: "--compress" +// NORDC: clang-linker-wrapper{{.*}}"--emit-fatbin-only"{{.*}}"--compress" + +// RUN: %clang -### --target=x86_64-linux-gnu \ +// RUN: -x hip --offload-arch=gfx1100 \ +// RUN: -fgpu-rdc -nogpuinc -nogpulib -c %S/Inputs/hip_multiple_inputs/a.cu \ +// RUN: --offload-compress --no-offload-compress \ +// RUN: 2>&1 | FileCheck %s --check-prefix=NOCOMPRESS + +// NOCOMPRESS: llvm-offload-binary{{.*}}"--image={{[^"]*}}arch=gfx1100{{[^"]*}}"{{$}} + +// --offload-compress is HIP only, so other offload packagers ignore it. +// RUN: %clang -### --target=x86_64-linux-gnu -x c -fopenmp \ +// RUN: --offload-arch=gfx1100 -nogpulib --offload-compress -c %s \ +// RUN: 2>&1 | FileCheck %s --check-prefix=OMP + +// OMP: llvm-offload-binary{{.*}}kind=openmp +// OMP-NOT: "--compress" diff --git a/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-compressed.c b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-compressed.c new file mode 100644 index 00000000000000..65b09ec2fbbbbe --- /dev/null +++ b/clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-compressed.c @@ -0,0 +1,25 @@ +// REQUIRES: x86-registered-target +// REQUIRES: zlib || zstd + +// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.elf.o + +// RUN: llvm-offload-binary -o %t.out --compress \ +// RUN: --image=file=%t.elf.o,kind=hip,triple=amdgpu9.0a-amd-amdhsa,arch=gfx90a \ +// RUN: --image=file=%t.elf.o,kind=hip,triple=amdgpu9.08-amd-amdhsa,arch=gfx908 +// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o \ +// RUN: -fembed-offload-object=%t.out +// RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ +// RUN: --linker-path=/usr/bin/ld %t.o -o a.out 2>&1 \ +// RUN: | FileCheck %s --check-prefixes=CHECK,HIP + +// RUN: llvm-offload-binary -o %t-lib.out --compress \ +// RUN: --image=file=%t.elf.o,kind=openmp,triple=amdgpu9.0a-amd-amdhsa,arch=gfx90a +// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t-lib.o \ +// RUN: -fembed-offload-object=%t-lib.out +// RUN: rm -f %t.a && llvm-ar rcs %t.a %t-lib.o +// RUN: clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu \ +// RUN: --linker-path=/usr/bin/ld --whole-archive %t.a --no-whole-archive \ +// RUN: -o a.out 2>&1 | FileCheck %s + +// CHECK: clang{{.*}} --target=amdgpu9.0a-amd-amdhsa -mcpu=gfx90a +// HIP: clang{{.*}} --target=amdgpu9.08-amd-amdhsa -mcpu=gfx908 diff --git a/llvm/docs/CommandGuide/llvm-offload-binary.md b/llvm/docs/CommandGuide/llvm-offload-binary.md index 6af4bbe7d4ffa3..7cca5be9736c38 100644 --- a/llvm/docs/CommandGuide/llvm-offload-binary.md +++ b/llvm/docs/CommandGuide/llvm-offload-binary.md @@ -69,11 +69,12 @@ Compress the packaged offload binary. ::: :::{option} --compression-format=<zstd|zlib> -Select the compression format. The default is `zstd`. +Select the compression format. The default is `zstd`, or `zlib` if LLVM was +built without zstd support. ::: :::{option} --compression-level=<level> -Specify a compression level for the selected format. +Specify a compression level. This requires {option}`--compression-format`, ::: :::{option} --image=<<key>=<value>,...> diff --git a/llvm/test/tools/llvm-offload-binary/compress-level-zlib.test b/llvm/test/tools/llvm-offload-binary/compress-level-zlib.test new file mode 100644 index 00000000000000..7f5fd63b7a2e30 --- /dev/null +++ b/llvm/test/tools/llvm-offload-binary/compress-level-zlib.test @@ -0,0 +1,13 @@ +# REQUIRES: zlib + +# RUN: llvm-offload-binary -o %t --compress --compression-format=zlib --compression-level=9 --image=file=%s,arch=abc,triple=x-y-z +# RUN: llvm-offload-binary %t --image=file=%t.out,arch=abc,triple=x-y-z +# RUN: diff %s %t.out +# RUN: llvm-offload-binary -o %t.default --compress --compression-format=zlib --compression-level=-1 --image=file=%s,arch=abc,triple=x-y-z +# RUN: llvm-offload-binary %t.default --image=file=%t.default.out,arch=abc,triple=x-y-z +# RUN: diff %s %t.default.out +# RUN: not llvm-offload-binary -o %t.bad --compress --compression-format=zlib --compression-level=12 --image=file=%s,arch=abc,triple=x-y-z 2>&1 | FileCheck %s +# RUN: not llvm-offload-binary -o %t.low --compress --compression-format=zlib --compression-level=-2 --image=file=%s,arch=abc,triple=x-y-z 2>&1 | FileCheck %s --check-prefix=LOW + +# CHECK: invalid zlib compression level 12; expected a value in [-1, 9] +# LOW: invalid zlib compression level -2; expected a value in [-1, 9] diff --git a/llvm/test/tools/llvm-offload-binary/compress.test b/llvm/test/tools/llvm-offload-binary/compress.test index 55232560ead2c5..35a858693e09fa 100644 --- a/llvm/test/tools/llvm-offload-binary/compress.test +++ b/llvm/test/tools/llvm-offload-binary/compress.test @@ -8,6 +8,20 @@ # RUN: llvm-offload-binary -o %t.neg --compress --compression-format=zstd --compression-level=-5 --image=file=%s,arch=abc,triple=x-y-z # RUN: llvm-offload-binary %t.neg --image=file=%t.neg.out,arch=abc,triple=x-y-z # RUN: diff %s %t.neg.out +# RUN: llvm-offload-binary -o %t.fast --compress --compression-format=zstd --compression-level=-10 --image=file=%s,arch=abc,triple=x-y-z +# RUN: llvm-offload-binary %t.fast --image=file=%t.fast.out,arch=abc,triple=x-y-z +# RUN: diff %s %t.fast.out + +# RUN: not llvm-offload-binary -o %t.nfmt --compress --compression-level=12 --image=file=%s,arch=abc,triple=x-y-z 2>&1 | FileCheck %s --check-prefix=NEED-FORMAT +# RUN: llvm-offload-binary -o %t.l12 --compress --compression-format=zstd --compression-level=12 --image=file=%s,arch=abc,triple=x-y-z +# RUN: llvm-offload-binary %t.l12 --image=file=%t.l12.out,arch=abc,triple=x-y-z +# RUN: diff %s %t.l12.out +# RUN: not llvm-offload-binary -o %t.bad --compress --compression-format=zstd --compression-level=100 --image=file=%s,arch=abc,triple=x-y-z 2>&1 | FileCheck %s --check-prefix=BAD-LEVEL +# RUN: not llvm-offload-binary -o %t.low --compress --compression-format=zstd --compression-level=-200000 --image=file=%s,arch=abc,triple=x-y-z 2>&1 | FileCheck %s --check-prefix=BAD-LOW + +# NEED-FORMAT: --compression-level requires --compression-format +# BAD-LEVEL: invalid zstd compression level 100; expected a value in [-131072, 22] +# BAD-LOW: invalid zstd compression level -200000; expected a value in [-131072, 22] # RUN: llvm-offload-binary -o %t.u --image=file=%s,arch=abc,triple=x-y-z # RUN: cat %t %t.u > %t.concat diff --git a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp index 35946b2b509c7d..1bdf61c483dc96 100644 --- a/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp +++ b/llvm/tools/llvm-offload-binary/llvm-offload-binary.cpp @@ -68,7 +68,8 @@ static cl::opt<compression::Format> CompressionFormat( static cl::opt<int> CompressionLevel("compression-level", - cl::desc("Compression level used with --compress"), + cl::desc("Compression level used with --compress. " + "Requires --compression-format"), cl::init(-1), cl::cat(OffloadBinaryCategory)); /// Path of the current binary. @@ -89,6 +90,31 @@ static DenseMap<StringRef, StringRef> getImageArguments(StringRef Image, return Args; } +static constexpr int ZlibMinCompressionLevel = -1; +static constexpr int ZstdMinCompressionLevel = -131072; +static constexpr int ZstdMaxCompressionLevel = 22; + +static Error checkCompressionLevel(compression::Format Format, int Level) { + int Lo, Hi; + const char *Name; + if (Format == compression::Format::Zlib) { + Lo = ZlibMinCompressionLevel; + Hi = compression::zlib::BestSizeCompression; + Name = "zlib"; + } else { + Lo = ZstdMinCompressionLevel; + Hi = ZstdMaxCompressionLevel; + Name = "zstd"; + } + if (Level < Lo || Level > Hi) + return createStringError( + inconvertibleErrorCode(), + "invalid %s compression level %d; expected a value " + "in [%d, %d]", + Name, Level, Lo, Hi); + return Error::success(); +} + static Error writeFile(StringRef Filename, StringRef Data) { Expected<std::unique_ptr<FileOutputBuffer>> OutputOrErr = FileOutputBuffer::create(Filename, Data.size()); @@ -143,12 +169,22 @@ static Error bundleImages() { SmallString<0> Buffer; if (Compress) { - if (const char *Reason = - compression::getReasonIfUnsupported(CompressionFormat)) + compression::Format Format = CompressionFormat; + if (!CompressionFormat.getNumOccurrences() && + !compression::zstd::isAvailable()) + Format = compression::Format::Zlib; + if (const char *Reason = compression::getReasonIfUnsupported(Format)) return createStringError(inconvertibleErrorCode(), Reason); - compression::Params Params(CompressionFormat); - if (CompressionLevel.getNumOccurrences()) + compression::Params Params(Format); + if (CompressionLevel.getNumOccurrences()) { + if (!CompressionFormat.getNumOccurrences()) + return createStringError( + inconvertibleErrorCode(), + "--compression-level requires --compression-format"); + if (Error E = checkCompressionLevel(Format, CompressionLevel)) + return E; Params.level = CompressionLevel; + } Expected<SmallString<0>> CompressedOrErr = OffloadBinary::write(AllImages, Params); if (!CompressedOrErr) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
