Author: michaelselehov Date: 2026-06-30T18:57:06+02:00 New Revision: 1c4c6f701ddd0645114f97b126654309a0d9dcb0
URL: https://github.com/llvm/llvm-project/commit/1c4c6f701ddd0645114f97b126654309a0d9dcb0 DIFF: https://github.com/llvm/llvm-project/commit/1c4c6f701ddd0645114f97b126654309a0d9dcb0.diff LOG: [Offload][NFC] Deduplicate CompressedOffloadBundle into llvm::object (#206741) The compressed offload bundle (CCOB) format had two byte-for-byte identical implementations: one in clang/lib/Driver/OffloadBundler.cpp and the canonical one in llvm/lib/Object/OffloadBundle.cpp (already used by llvm-objdump, comgr and others). Remove the Clang-side copy (class declaration, compress/decompress/ tryParse, the RawCompressedBundleHeader union, getHeaderSize, the formatWithCommas helper and the private timer group) and route the driver through llvm::object::CompressedOffloadBundle, which is already linked via the Object component. The only API difference is the verbose-reporting channel (bool Verbose vs. raw_ostream *VerboseStream); the driver now passes &llvm::errs() when verbose output is requested, preserving the previous behavior. No functional change intended. Co-authored-by: mselehov <[email protected]> Added: Modified: clang/include/clang/Driver/OffloadBundler.h clang/lib/Driver/OffloadBundler.cpp Removed: ################################################################################ diff --git a/clang/include/clang/Driver/OffloadBundler.h b/clang/include/clang/Driver/OffloadBundler.h index 637c9d146533f..96839240543af 100644 --- a/clang/include/clang/Driver/OffloadBundler.h +++ b/clang/include/clang/Driver/OffloadBundler.h @@ -92,42 +92,6 @@ struct OffloadTargetInfo { std::string str() const; }; -// CompressedOffloadBundle represents the format for the compressed offload -// bundles. -// -// The format is as follows: -// - Magic Number (4 bytes) - A constant "CCOB". -// - Version (2 bytes) -// - Compression Method (2 bytes) - Uses the values from -// llvm::compression::Format. -// - Total file size (4 bytes in V2, 8 bytes in V3). -// - Uncompressed Size (4 bytes in V1/V2, 8 bytes in V3). -// - Truncated MD5 Hash (8 bytes). -// - Compressed Data (variable length). -class CompressedOffloadBundle { -private: - static inline const llvm::StringRef MagicNumber = "CCOB"; - -public: - struct CompressedBundleHeader { - unsigned Version; - llvm::compression::Format CompressionFormat; - std::optional<size_t> FileSize; - size_t UncompressedFileSize; - uint64_t Hash; - - static llvm::Expected<CompressedBundleHeader> tryParse(llvm::StringRef); - }; - - static inline const uint16_t DefaultVersion = 3; - - static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> - compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, - uint16_t Version, bool Verbose = false); - static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> - decompress(const llvm::MemoryBuffer &Input, bool Verbose = false); -}; - /// Check whether the bundle id is in the following format: /// <kind>-<triple>[-<target id>[:target features]] /// <triple> := <arch>-<vendor>-<os>-<env> diff --git a/clang/lib/Driver/OffloadBundler.cpp b/clang/lib/Driver/OffloadBundler.cpp index 9d6f32c4a4e8f..4c7d13212f968 100644 --- a/clang/lib/Driver/OffloadBundler.cpp +++ b/clang/lib/Driver/OffloadBundler.cpp @@ -28,6 +28,7 @@ #include "llvm/Object/ArchiveWriter.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Object/OffloadBundle.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Compression.h" @@ -65,18 +66,6 @@ using namespace llvm; using namespace llvm::object; using namespace clang; -namespace { -struct CreateClangOffloadBundlerTimerGroup { - static void *call() { - return new TimerGroup("Clang Offload Bundler Timer Group", - "Timer group for clang offload bundler"); - } -}; -} // namespace -static llvm::ManagedStatic<llvm::TimerGroup, - CreateClangOffloadBundlerTimerGroup> - ClangOffloadBundlerTimerGroup; - /// Magic string that marks the existence of offloading data. #define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" @@ -1013,336 +1002,6 @@ OffloadBundlerConfig::OffloadBundlerConfig() } } -// Utility function to format numbers with commas -static std::string formatWithCommas(unsigned long long Value) { - std::string Num = std::to_string(Value); - int InsertPosition = Num.length() - 3; - while (InsertPosition > 0) { - Num.insert(InsertPosition, ","); - InsertPosition -= 3; - } - return Num; -} - -llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> -CompressedOffloadBundle::compress(llvm::compression::Params P, - const llvm::MemoryBuffer &Input, - uint16_t Version, bool Verbose) { - if (!llvm::compression::zstd::isAvailable() && - !llvm::compression::zlib::isAvailable()) - return createStringError(llvm::inconvertibleErrorCode(), - "Compression not supported"); - llvm::Timer HashTimer("Hash Calculation Timer", "Hash calculation time", - *ClangOffloadBundlerTimerGroup); - if (Verbose) - HashTimer.startTimer(); - llvm::MD5 Hash; - llvm::MD5::MD5Result Result; - Hash.update(Input.getBuffer()); - Hash.final(Result); - uint64_t TruncatedHash = Result.low(); - if (Verbose) - HashTimer.stopTimer(); - - SmallVector<uint8_t, 0> CompressedBuffer; - auto BufferUint8 = llvm::ArrayRef<uint8_t>( - reinterpret_cast<const uint8_t *>(Input.getBuffer().data()), - Input.getBuffer().size()); - llvm::Timer CompressTimer("Compression Timer", "Compression time", - *ClangOffloadBundlerTimerGroup); - if (Verbose) - CompressTimer.startTimer(); - llvm::compression::compress(P, BufferUint8, CompressedBuffer); - if (Verbose) - CompressTimer.stopTimer(); - - uint16_t CompressionMethod = static_cast<uint16_t>(P.format); - - // Store sizes in 64-bit variables first - uint64_t UncompressedSize64 = Input.getBuffer().size(); - uint64_t TotalFileSize64; - - // Calculate total file size based on version - if (Version == 2) { - // For V2, ensure the sizes don't exceed 32-bit limit - if (UncompressedSize64 > std::numeric_limits<uint32_t>::max()) - return createStringError(llvm::inconvertibleErrorCode(), - "Uncompressed size exceeds version 2 limit"); - if ((MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) + - sizeof(CompressionMethod) + sizeof(uint32_t) + sizeof(TruncatedHash) + - CompressedBuffer.size()) > std::numeric_limits<uint32_t>::max()) - return createStringError(llvm::inconvertibleErrorCode(), - "Total file size exceeds version 2 limit"); - - TotalFileSize64 = MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) + - sizeof(CompressionMethod) + sizeof(uint32_t) + - sizeof(TruncatedHash) + CompressedBuffer.size(); - } else { // Version 3 - TotalFileSize64 = MagicNumber.size() + sizeof(uint64_t) + sizeof(Version) + - sizeof(CompressionMethod) + sizeof(uint64_t) + - sizeof(TruncatedHash) + CompressedBuffer.size(); - } - - SmallVector<char, 0> FinalBuffer; - llvm::raw_svector_ostream OS(FinalBuffer); - OS << MagicNumber; - OS.write(reinterpret_cast<const char *>(&Version), sizeof(Version)); - OS.write(reinterpret_cast<const char *>(&CompressionMethod), - sizeof(CompressionMethod)); - - // Write size fields according to version - if (Version == 2) { - uint32_t TotalFileSize32 = static_cast<uint32_t>(TotalFileSize64); - uint32_t UncompressedSize32 = static_cast<uint32_t>(UncompressedSize64); - OS.write(reinterpret_cast<const char *>(&TotalFileSize32), - sizeof(TotalFileSize32)); - OS.write(reinterpret_cast<const char *>(&UncompressedSize32), - sizeof(UncompressedSize32)); - } else { // Version 3 - OS.write(reinterpret_cast<const char *>(&TotalFileSize64), - sizeof(TotalFileSize64)); - OS.write(reinterpret_cast<const char *>(&UncompressedSize64), - sizeof(UncompressedSize64)); - } - - OS.write(reinterpret_cast<const char *>(&TruncatedHash), - sizeof(TruncatedHash)); - OS.write(reinterpret_cast<const char *>(CompressedBuffer.data()), - CompressedBuffer.size()); - - if (Verbose) { - auto MethodUsed = - P.format == llvm::compression::Format::Zstd ? "zstd" : "zlib"; - double CompressionRate = - static_cast<double>(UncompressedSize64) / CompressedBuffer.size(); - double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime(); - double CompressionSpeedMBs = - (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds; - llvm::errs() << "Compressed bundle format version: " << Version << "\n" - << "Total file size (including headers): " - << formatWithCommas(TotalFileSize64) << " bytes\n" - << "Compression method used: " << MethodUsed << "\n" - << "Compression level: " << P.level << "\n" - << "Binary size before compression: " - << formatWithCommas(UncompressedSize64) << " bytes\n" - << "Binary size after compression: " - << formatWithCommas(CompressedBuffer.size()) << " bytes\n" - << "Compression rate: " - << llvm::format("%.2lf", CompressionRate) << "\n" - << "Compression ratio: " - << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n" - << "Compression speed: " - << llvm::format("%.2lf MB/s", CompressionSpeedMBs) << "\n" - << "Truncated MD5 hash: " - << llvm::format_hex(TruncatedHash, 16) << "\n"; - } - - return llvm::MemoryBuffer::getMemBufferCopy( - llvm::StringRef(FinalBuffer.data(), FinalBuffer.size())); -} - -// Use packed structs to avoid padding, such that the structs map the serialized -// format. -LLVM_PACKED_START -union RawCompressedBundleHeader { - struct CommonFields { - uint32_t Magic; - uint16_t Version; - uint16_t Method; - }; - - struct V1Header { - CommonFields Common; - uint32_t UncompressedFileSize; - uint64_t Hash; - }; - - struct V2Header { - CommonFields Common; - uint32_t FileSize; - uint32_t UncompressedFileSize; - uint64_t Hash; - }; - - struct V3Header { - CommonFields Common; - uint64_t FileSize; - uint64_t UncompressedFileSize; - uint64_t Hash; - }; - - CommonFields Common; - V1Header V1; - V2Header V2; - V3Header V3; -}; -LLVM_PACKED_END - -// Helper method to get header size based on version -static size_t getHeaderSize(uint16_t Version) { - switch (Version) { - case 1: - return sizeof(RawCompressedBundleHeader::V1Header); - case 2: - return sizeof(RawCompressedBundleHeader::V2Header); - case 3: - return sizeof(RawCompressedBundleHeader::V3Header); - default: - llvm_unreachable("Unsupported version"); - } -} - -Expected<CompressedOffloadBundle::CompressedBundleHeader> -CompressedOffloadBundle::CompressedBundleHeader::tryParse(StringRef Blob) { - assert(Blob.size() >= sizeof(RawCompressedBundleHeader::CommonFields)); - assert(llvm::identify_magic(Blob) == - llvm::file_magic::offload_bundle_compressed); - - RawCompressedBundleHeader Header; - memcpy(&Header, Blob.data(), std::min(Blob.size(), sizeof(Header))); - - CompressedBundleHeader Normalized; - Normalized.Version = Header.Common.Version; - - size_t RequiredSize = getHeaderSize(Normalized.Version); - if (Blob.size() < RequiredSize) - return createStringError(inconvertibleErrorCode(), - "Compressed bundle header size too small"); - - switch (Normalized.Version) { - case 1: - Normalized.UncompressedFileSize = Header.V1.UncompressedFileSize; - Normalized.Hash = Header.V1.Hash; - break; - case 2: - Normalized.FileSize = Header.V2.FileSize; - Normalized.UncompressedFileSize = Header.V2.UncompressedFileSize; - Normalized.Hash = Header.V2.Hash; - break; - case 3: - Normalized.FileSize = Header.V3.FileSize; - Normalized.UncompressedFileSize = Header.V3.UncompressedFileSize; - Normalized.Hash = Header.V3.Hash; - break; - default: - return createStringError(inconvertibleErrorCode(), - "Unknown compressed bundle version"); - } - - // Determine compression format - switch (Header.Common.Method) { - case static_cast<uint16_t>(compression::Format::Zlib): - case static_cast<uint16_t>(compression::Format::Zstd): - Normalized.CompressionFormat = - static_cast<compression::Format>(Header.Common.Method); - break; - default: - return createStringError(inconvertibleErrorCode(), - "Unknown compressing method"); - } - - return Normalized; -} - -llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>> -CompressedOffloadBundle::decompress(const llvm::MemoryBuffer &Input, - bool Verbose) { - StringRef Blob = Input.getBuffer(); - - // Check minimum header size (using V1 as it's the smallest) - if (Blob.size() < sizeof(RawCompressedBundleHeader::CommonFields)) - return llvm::MemoryBuffer::getMemBufferCopy(Blob); - - if (llvm::identify_magic(Blob) != - llvm::file_magic::offload_bundle_compressed) { - if (Verbose) - llvm::errs() << "Uncompressed bundle.\n"; - return llvm::MemoryBuffer::getMemBufferCopy(Blob); - } - - Expected<CompressedBundleHeader> HeaderOrErr = - CompressedBundleHeader::tryParse(Blob); - if (!HeaderOrErr) - return HeaderOrErr.takeError(); - - const CompressedBundleHeader &Normalized = *HeaderOrErr; - unsigned ThisVersion = Normalized.Version; - size_t HeaderSize = getHeaderSize(ThisVersion); - - llvm::compression::Format CompressionFormat = Normalized.CompressionFormat; - - size_t TotalFileSize = Normalized.FileSize.value_or(0); - size_t UncompressedSize = Normalized.UncompressedFileSize; - auto StoredHash = Normalized.Hash; - - llvm::Timer DecompressTimer("Decompression Timer", "Decompression time", - *ClangOffloadBundlerTimerGroup); - if (Verbose) - DecompressTimer.startTimer(); - - SmallVector<uint8_t, 0> DecompressedData; - StringRef CompressedData = - Blob.substr(HeaderSize, TotalFileSize - HeaderSize); - if (llvm::Error DecompressionError = llvm::compression::decompress( - CompressionFormat, llvm::arrayRefFromStringRef(CompressedData), - DecompressedData, UncompressedSize)) - return createStringError(inconvertibleErrorCode(), - "Could not decompress embedded file contents: " + - llvm::toString(std::move(DecompressionError))); - - if (Verbose) { - DecompressTimer.stopTimer(); - - double DecompressionTimeSeconds = - DecompressTimer.getTotalTime().getWallTime(); - - // Recalculate MD5 hash for integrity check - llvm::Timer HashRecalcTimer("Hash Recalculation Timer", - "Hash recalculation time", - *ClangOffloadBundlerTimerGroup); - HashRecalcTimer.startTimer(); - llvm::MD5 Hash; - llvm::MD5::MD5Result Result; - Hash.update(llvm::ArrayRef<uint8_t>(DecompressedData)); - Hash.final(Result); - uint64_t RecalculatedHash = Result.low(); - HashRecalcTimer.stopTimer(); - bool HashMatch = (StoredHash == RecalculatedHash); - - double CompressionRate = - static_cast<double>(UncompressedSize) / CompressedData.size(); - double DecompressionSpeedMBs = - (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds; - - llvm::errs() << "Compressed bundle format version: " << ThisVersion << "\n"; - if (ThisVersion >= 2) - llvm::errs() << "Total file size (from header): " - << formatWithCommas(TotalFileSize) << " bytes\n"; - llvm::errs() << "Decompression method: " - << (CompressionFormat == llvm::compression::Format::Zlib - ? "zlib" - : "zstd") - << "\n" - << "Size before decompression: " - << formatWithCommas(CompressedData.size()) << " bytes\n" - << "Size after decompression: " - << formatWithCommas(UncompressedSize) << " bytes\n" - << "Compression rate: " - << llvm::format("%.2lf", CompressionRate) << "\n" - << "Compression ratio: " - << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n" - << "Decompression speed: " - << llvm::format("%.2lf MB/s", DecompressionSpeedMBs) << "\n" - << "Stored hash: " << llvm::format_hex(StoredHash, 16) << "\n" - << "Recalculated hash: " - << llvm::format_hex(RecalculatedHash, 16) << "\n" - << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n"; - } - - return llvm::MemoryBuffer::getMemBufferCopy( - llvm::toStringRef(DecompressedData)); -} - // List bundle IDs. Return true if an error was found. Error OffloadBundler::ListBundleIDsInFile( StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) { @@ -1380,7 +1039,8 @@ Error OffloadBundler::ListBundleIDsInFile( // Decompress the input if necessary. Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr = - CompressedOffloadBundle::decompress(**CodeOrErr, BundlerConfig.Verbose); + CompressedOffloadBundle::decompress( + **CodeOrErr, BundlerConfig.Verbose ? &llvm::errs() : nullptr); if (!DecompressedBufferOrErr) return createStringError( inconvertibleErrorCode(), @@ -1562,7 +1222,7 @@ Error OffloadBundler::BundleFiles() { {BundlerConfig.CompressionFormat, BundlerConfig.CompressionLevel, /*zstdEnableLdm=*/true}, *BufferMemory, BundlerConfig.CompressedBundleVersion, - BundlerConfig.Verbose); + BundlerConfig.Verbose ? &llvm::errs() : nullptr); if (auto Error = CompressionResult.takeError()) return Error; @@ -1633,7 +1293,8 @@ Error OffloadBundler::UnbundleFiles() { // Decompress the blob if necessary. Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr = - CompressedOffloadBundle::decompress(**BlobOrErr, BundlerConfig.Verbose); + CompressedOffloadBundle::decompress( + **BlobOrErr, BundlerConfig.Verbose ? &llvm::errs() : nullptr); if (!DecompressedBufferOrErr) return createStringError( inconvertibleErrorCode(), @@ -1928,8 +1589,9 @@ Error OffloadBundler::UnbundleArchive() { // Decompress the buffer if necessary. Expected<std::unique_ptr<MemoryBuffer>> DecompressedBufferOrErr = - CompressedOffloadBundle::decompress(*TempCodeObjectBuffer, - BundlerConfig.Verbose); + CompressedOffloadBundle::decompress( + *TempCodeObjectBuffer, + BundlerConfig.Verbose ? &llvm::errs() : nullptr); if (!DecompressedBufferOrErr) return createStringError( inconvertibleErrorCode(), _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
