llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-lldb Author: satyanarayana reddy janga (satyajanga) <details> <summary>Changes</summary> ## Summary Clang supports embedding binaries using the clang offload bundler: https://clang.llvm.org/docs/ClangOffloadBundler.html This patch creates an ObjectContainer plug-in that allows us to get the contained binaries within any executable. ### why we need this? AMD/HIP host binaries may contain a .hip_fatbin section holding a Clang offload bundle with one or more AMDGPU ELF code objects, typically for different GPU architectures. Each ELF code object can contain multiple ml kernels. ### Test Added tests for the logic. --- Patch is 32.00 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222362.diff 7 Files Affected: - (modified) lldb/source/Plugins/ObjectContainer/CMakeLists.txt (+1) - (added) lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/CMakeLists.txt (+14) - (added) lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.cpp (+263) - (added) lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.h (+79) - (modified) lldb/source/Symbol/ObjectFile.cpp (+14-7) - (modified) lldb/unittests/ObjectContainer/CMakeLists.txt (+3) - (added) lldb/unittests/ObjectContainer/ObjectContainerClangOffloadBundleTest.cpp (+401) ``````````diff diff --git a/lldb/source/Plugins/ObjectContainer/CMakeLists.txt b/lldb/source/Plugins/ObjectContainer/CMakeLists.txt index 220a76d1159d5..8cf7157cd0ad5 100644 --- a/lldb/source/Plugins/ObjectContainer/CMakeLists.txt +++ b/lldb/source/Plugins/ObjectContainer/CMakeLists.txt @@ -2,5 +2,6 @@ set_property(DIRECTORY PROPERTY LLDB_PLUGIN_KIND ObjectContainer) add_subdirectory(BSD-Archive) add_subdirectory(Big-Archive) +add_subdirectory(Clang-Offload-Bundle) add_subdirectory(Universal-Mach-O) add_subdirectory(Mach-O-Fileset) diff --git a/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/CMakeLists.txt b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/CMakeLists.txt new file mode 100644 index 0000000000000..2694e4e828914 --- /dev/null +++ b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/CMakeLists.txt @@ -0,0 +1,14 @@ +add_lldb_library(lldbPluginObjectContainerClangOffloadBundle PLUGIN + ObjectContainerClangOffloadBundle.cpp + + LINK_COMPONENTS + BinaryFormat + Object + TargetParser + LINK_LIBS + lldbCore + lldbHost + lldbSymbol + lldbTarget + lldbUtility + ) diff --git a/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.cpp b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.cpp new file mode 100644 index 0000000000000..5280210f762ca --- /dev/null +++ b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.cpp @@ -0,0 +1,263 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ObjectContainerClangOffloadBundle.h" + +#include "lldb/Core/Module.h" +#include "lldb/Core/ModuleSpec.h" +#include "lldb/Core/PluginManager.h" +#include "lldb/Host/FileSystem.h" +#include "lldb/Symbol/ObjectFile.h" +#include "lldb/Target/Target.h" +#include "lldb/Utility/DataBuffer.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/Log.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/OffloadBundle.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/TargetParser/AMDGPUTargetParser.h" + +#include <limits> +#include <string> +#include <utility> + +using namespace lldb; +using namespace lldb_private; + +LLDB_PLUGIN_DEFINE(ObjectContainerClangOffloadBundle) + +void ObjectContainerClangOffloadBundle::Initialize() { + PluginManager::RegisterPlugin( + GetPluginNameStatic(), GetPluginDescriptionStatic(), CreateInstance, + GetModuleSpecifications, /*create_memory_callback=*/nullptr); +} + +void ObjectContainerClangOffloadBundle::Terminate() { + PluginManager::UnregisterPlugin(CreateInstance); +} + +ObjectContainerClangOffloadBundle::ObjectContainerClangOffloadBundle( + const ModuleSP &module_sp, DataBufferSP &data_sp, + lldb::offset_t data_offset, const FileSpec *file, + lldb::offset_t file_offset, lldb::offset_t length) + : ObjectContainer(module_sp, file, file_offset, length, data_sp, + data_offset) {} + +ObjectContainerClangOffloadBundle::~ObjectContainerClangOffloadBundle() = + default; + +bool ObjectContainerClangOffloadBundle::MagicBytesMatch( + const DataExtractor &data) { + llvm::StringRef bytes(reinterpret_cast<const char *>(data.GetDataStart()), + data.GetByteSize()); + switch (llvm::identify_magic(bytes)) { + case llvm::file_magic::elf: + case llvm::file_magic::elf_relocatable: + case llvm::file_magic::elf_executable: + case llvm::file_magic::elf_shared_object: + case llvm::file_magic::elf_core: + return true; + default: + return false; + } +} + +static ArchSpec ParseArchFromBundleEntryID(llvm::StringRef id) { + // Bundle entry IDs contain an offload kind, a target triple, and optionally + // a target ID and target features. + llvm::StringRef triple = id.split('-').second; + // AMDGPU target features follow the processor name after a colon. ArchSpec + // models the processor, but not these target features. + triple = triple.split(':').first; + if (triple.empty()) + return {}; + + // Clang offload bundle IDs can use legacy or processor-qualified + // architecture spellings, while LLDB's canonical name is "amdgpu". + auto [architecture, triple_suffix] = triple.split('-'); + llvm::Triple target_triple(triple); + if (target_triple.isAMDGCN()) { + llvm::StringRef processor = + llvm::AMDGPU::getArchNameFromSubArch(target_triple.getSubArch()); + if (!processor.empty()) { + std::string normalized_triple = "amdgpu-"; + normalized_triple.append(triple_suffix.split("--").first); + normalized_triple.append("--"); + normalized_triple.append(processor); + return ArchSpec(normalized_triple); + } + } + if (architecture == "amdgcn") { + std::string normalized_triple = "amdgpu-"; + normalized_triple.append(triple_suffix); + return ArchSpec(normalized_triple); + } + return ArchSpec(triple); +} + +bool ObjectContainerClangOffloadBundle::FindBundleEntries( + const FileSpec &file, DataExtractorSP extractor_sp, + lldb::offset_t file_offset, lldb::offset_t file_size, + std::vector<Entry> &entries) { + const uint8_t *data = nullptr; + size_t data_size = 0; + DataBufferSP mapped_data_sp; + if (extractor_sp && extractor_sp->HasData() && + extractor_sp->GetByteSize() >= file_size) { + data = extractor_sp->GetDataStart(); + data_size = file_size ? file_size : extractor_sp->GetByteSize(); + } else if (file) { + mapped_data_sp = + FileSystem::Instance().CreateDataBuffer(file, file_size, file_offset); + if (mapped_data_sp) { + data = mapped_data_sp->GetBytes(); + data_size = mapped_data_sp->GetByteSize(); + } + } + if (!data) + return false; + + llvm::StringRef bytes(reinterpret_cast<const char *>(data), data_size); + const std::string path = file.GetPath(); + llvm::MemoryBufferRef buffer(bytes, path); + auto object_or_error = llvm::object::ObjectFile::createObjectFile(buffer); + if (!object_or_error) { + LLDB_LOG_ERROR(GetLog(LLDBLog::Object), object_or_error.takeError(), + "unable to parse clang offload bundle container: {0}"); + return false; + } + + llvm::SmallVector<llvm::object::OffloadBundleFatBin> bundles; + if (llvm::Error error = llvm::object::extractOffloadBundleFatBinary( + **object_or_error, bundles)) { + LLDB_LOG_ERROR(GetLog(LLDBLog::Object), std::move(error), + "unable to extract clang offload bundle: {0}"); + return false; + } + + for (llvm::object::OffloadBundleFatBin &bundle : bundles) { + // Compressed bundle entries refer to the decompressed buffer, not to + // offsets in the containing object file. + if (bundle.isDecompressed()) { + LLDB_LOG(GetLog(LLDBLog::Object), + "skipping compressed clang offload bundle in {0}", path); + continue; + } + + for (const llvm::object::OffloadBundleEntry &bundle_entry : + bundle.getEntries()) { + if (bundle_entry.Size == 0 || bundle_entry.Offset == 0 || + bundle_entry.Offset > data_size || + bundle_entry.Size > data_size - bundle_entry.Offset || + bundle_entry.Offset > + std::numeric_limits<lldb::offset_t>::max() - file_offset || + bundle_entry.Size > std::numeric_limits<lldb::offset_t>::max() - + file_offset - bundle_entry.Offset) + continue; + + ArchSpec arch = ParseArchFromBundleEntryID(bundle_entry.ID); + if (!arch.IsValid()) + continue; + + entries.push_back({std::move(arch), file_offset + bundle_entry.Offset, + bundle_entry.Size}); + } + } + + return !entries.empty(); +} + +ObjectContainer *ObjectContainerClangOffloadBundle::CreateInstance( + const ModuleSP &module_sp, DataBufferSP &data_sp, + lldb::offset_t data_offset, const FileSpec *file, + lldb::offset_t file_offset, lldb::offset_t length) { + if (!data_sp || !file) + return nullptr; + + DataExtractor data; + data.SetData(data_sp, data_offset, length); + if (!MagicBytesMatch(data)) + return nullptr; + + auto container_up = std::make_unique<ObjectContainerClangOffloadBundle>( + module_sp, data_sp, data_offset, file, file_offset, length); + if (!container_up->ParseHeader()) + return nullptr; + return container_up.release(); +} + +bool ObjectContainerClangOffloadBundle::ParseHeader() { + m_entries.clear(); + return FindBundleEntries(m_file, m_extractor_sp, m_offset, m_length, + m_entries); +} + +size_t ObjectContainerClangOffloadBundle::GetNumArchitectures() const { + return m_entries.size(); +} + +bool ObjectContainerClangOffloadBundle::GetArchitectureAtIndex( + uint32_t idx, ArchSpec &arch) const { + if (idx >= m_entries.size()) + return false; + arch = m_entries[idx].arch; + return true; +} + +ModuleSpecList ObjectContainerClangOffloadBundle::GetModuleSpecifications( + const FileSpec &file, DataExtractorSP &extractor_sp, + lldb::offset_t file_offset, lldb::offset_t file_size) { + if (!extractor_sp || !MagicBytesMatch(*extractor_sp)) + return {}; + + std::vector<Entry> entries; + if (!FindBundleEntries(file, extractor_sp, file_offset, file_size, entries)) + return {}; + + ModuleSpecList specs; + for (const Entry &entry : entries) { + ModuleSpec spec(file, entry.arch); + spec.SetObjectOffset(entry.offset); + spec.SetObjectSize(entry.size); + specs.Append(spec); + } + return specs; +} + +ObjectFileSP +ObjectContainerClangOffloadBundle::GetObjectFile(const FileSpec *file) { + ModuleSP module_sp = GetModule(); + if (!module_sp) + return {}; + + ArchSpec arch = module_sp->GetArchitecture(); + if (!arch.IsValid()) { + arch = Target::GetDefaultArchitecture(); + if (!arch.IsValid()) + arch.SetTriple(LLDB_ARCH_DEFAULT); + } + + for (int pass = 0; pass < 2; ++pass) { + for (const Entry &entry : m_entries) { + const bool matches = pass == 0 ? arch.IsExactMatch(entry.arch) + : arch.IsCompatibleMatch(entry.arch); + if (!matches) + continue; + + DataExtractorSP extractor_sp; + lldb::offset_t data_offset = 0; + if (ObjectFileSP object_file_sp = + ObjectFile::FindPlugin(module_sp, file, entry.offset, entry.size, + extractor_sp, data_offset)) + return object_file_sp; + } + } + + return {}; +} diff --git a/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.h b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.h new file mode 100644 index 0000000000000..1d63bcfda3232 --- /dev/null +++ b/lldb/source/Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.h @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_SOURCE_PLUGINS_OBJECTCONTAINER_CLANG_OFFLOAD_BUNDLE_OBJECTCONTAINERCLANGOFFLOADBUNDLE_H +#define LLDB_SOURCE_PLUGINS_OBJECTCONTAINER_CLANG_OFFLOAD_BUNDLE_OBJECTCONTAINERCLANGOFFLOADBUNDLE_H + +#include "lldb/Symbol/ObjectContainer.h" +#include "lldb/Utility/ArchSpec.h" +#include "lldb/Utility/FileSpec.h" + +#include <vector> + +class ObjectContainerClangOffloadBundle : public lldb_private::ObjectContainer { +public: + ObjectContainerClangOffloadBundle(const lldb::ModuleSP &module_sp, + lldb::DataBufferSP &data_sp, + lldb::offset_t data_offset, + const lldb_private::FileSpec *file, + lldb::offset_t file_offset, + lldb::offset_t length); + + ~ObjectContainerClangOffloadBundle() override; + + static void Initialize(); + static void Terminate(); + + static llvm::StringRef GetPluginNameStatic() { + return "clang-offload-bundle"; + } + + static llvm::StringRef GetPluginDescriptionStatic() { + return "Clang offload bundle object container reader for ELF files."; + } + + static lldb_private::ObjectContainer * + CreateInstance(const lldb::ModuleSP &module_sp, lldb::DataBufferSP &data_sp, + lldb::offset_t data_offset, const lldb_private::FileSpec *file, + lldb::offset_t file_offset, lldb::offset_t length); + + static lldb_private::ModuleSpecList + GetModuleSpecifications(const lldb_private::FileSpec &file, + lldb::DataExtractorSP &extractor_sp, + lldb::offset_t file_offset, lldb::offset_t file_size); + + static bool MagicBytesMatch(const lldb_private::DataExtractor &data); + + bool ParseHeader() override; + + size_t GetNumArchitectures() const override; + + bool GetArchitectureAtIndex(uint32_t idx, + lldb_private::ArchSpec &arch) const override; + + lldb::ObjectFileSP GetObjectFile(const lldb_private::FileSpec *file) override; + + llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); } + +protected: + struct Entry { + lldb_private::ArchSpec arch; + uint64_t offset = 0; + uint64_t size = 0; + }; + + std::vector<Entry> m_entries; + + static bool FindBundleEntries(const lldb_private::FileSpec &file, + lldb::DataExtractorSP extractor_sp, + lldb::offset_t file_offset, + lldb::offset_t file_size, + std::vector<Entry> &entries); +}; + +#endif // LLDB_SOURCE_PLUGINS_OBJECTCONTAINER_CLANG_OFFLOAD_BUNDLE_OBJECTCONTAINERCLANGOFFLOADBUNDLE_H diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp index 89d01568de926..ce30d2e92d13e 100644 --- a/lldb/source/Symbol/ObjectFile.cpp +++ b/lldb/source/Symbol/ObjectFile.cpp @@ -220,22 +220,29 @@ ModuleSpecList ObjectFile::GetModuleSpecifications( ModuleSpecList ObjectFile::GetModuleSpecifications( const lldb_private::FileSpec &file, lldb::DataExtractorSP &extractor_sp, lldb::offset_t file_offset, lldb::offset_t file_size) { + ModuleSpecList specs; + + // A container can be embedded in an otherwise valid object file. Preserve + // the object file's specifications and also query the container plug-ins. + // Try the ObjectFile plug-ins for (auto &cbs : PluginManager::GetObjectFileCallbacks()) { - ModuleSpecList specs = cbs.get_module_specifications( + ModuleSpecList object_specs = cbs.get_module_specifications( file, extractor_sp, file_offset, file_size); - if (specs.GetSize() > 0) - return specs; + if (object_specs.GetSize() > 0) { + specs.Append(object_specs); + break; + } } // Try the ObjectContainer plug-ins for (auto &cbs : PluginManager::GetObjectContainerCallbacks()) { - ModuleSpecList specs = cbs.get_module_specifications( + ModuleSpecList container_specs = cbs.get_module_specifications( file, extractor_sp, file_offset, file_size); - if (specs.GetSize() > 0) - return specs; + if (container_specs.GetSize() > 0) + specs.Append(container_specs); } - return {}; + return specs; } ObjectFile::ObjectFile(const lldb::ModuleSP &module_sp, diff --git a/lldb/unittests/ObjectContainer/CMakeLists.txt b/lldb/unittests/ObjectContainer/CMakeLists.txt index 8acfccf12f4dd..4df5d9482547c 100644 --- a/lldb/unittests/ObjectContainer/CMakeLists.txt +++ b/lldb/unittests/ObjectContainer/CMakeLists.txt @@ -1,9 +1,12 @@ add_lldb_unittest(ObjectContainerTests + ObjectContainerClangOffloadBundleTest.cpp ObjectContainerUniversalMachOTest.cpp LINK_LIBS + lldbPluginObjectContainerClangOffloadBundle lldbPluginObjectContainerMachOArchive lldbPluginObjectContainerMachOFileset + lldbPluginObjectFileELF lldbCore lldbUtilityHelpers LLVMTestingSupport diff --git a/lldb/unittests/ObjectContainer/ObjectContainerClangOffloadBundleTest.cpp b/lldb/unittests/ObjectContainer/ObjectContainerClangOffloadBundleTest.cpp new file mode 100644 index 0000000000000..46cdab7b48fd6 --- /dev/null +++ b/lldb/unittests/ObjectContainer/ObjectContainerClangOffloadBundleTest.cpp @@ -0,0 +1,401 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Plugins/ObjectContainer/Clang-Offload-Bundle/ObjectContainerClangOffloadBundle.h" +#include "Plugins/ObjectFile/ELF/ObjectFileELF.h" +#include "TestingSupport/SubsystemRAII.h" +#include "TestingSupport/TestUtilities.h" +#include "lldb/Core/Module.h" +#include "lldb/Core/ModuleSpec.h" +#include "lldb/Host/FileSystem.h" +#include "lldb/Host/HostInfo.h" +#include "lldb/Symbol/ObjectFile.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/FileUtilities.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" + +#include <array> +#include <cstdint> +#include <limits> +#include <optional> +#include <string> +#include <vector> + +using namespace lldb; +using namespace lldb_private; + +namespace { + +constexpr uint64_t BundleSectionOffset = 0x1000; +constexpr llvm::StringLiteral BundleMagic = "__CLANG_OFFLOAD_BUNDLE__"; + +void AppendU64(std::vector<uint8_t> &bytes, uint64_t value) { + for (unsigned i = 0; i != 8; ++i) + bytes.push_back(static_cast<uint8_t>(value >> (i * 8))); +} + +std::vector<uint8_t> MakeBundle(llvm::StringRef id, + llvm::ArrayRef<uint8_t> payload, + std::optional<uint64_t> entry_offset = {}) { + std::vector<uint8_t> bytes(BundleMagic.bytes_begin(), + BundleMagic.bytes_end()); + AppendU64(bytes, 1); + const uint64_t payload_offset = entry_offset.value_or( + BundleMagic.size() + 4 * sizeof(uint64_t) + id.size()); + AppendU64(bytes, payload_offset); + AppendU64(bytes, payload.size()); + AppendU64(bytes, id.size()); + bytes.insert(bytes.end(), id.bytes_begin(), id.bytes_end()); + if (!entry_offset) + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +struct BundleInput { + llvm::StringRef id; + llvm::ArrayRef<uint8_t> payload; +}; + +struct BundleData { + std::vector<uint8_t> bytes; + std::vector<uint64_t> entry_offsets; +}; + +BundleData MakeBundle(llvm::ArrayRef<BundleInput> inputs) { + BundleData result; + result.bytes.insert(result.bytes.end(), BundleMagic.bytes_begin(), + BundleMagic.bytes_end()); + AppendU64(result.bytes, inputs.size()); + + uint64_t payload_offset = BundleMagic.size() + sizeof(uint64_t); + for (const BundleInput &input : inputs) + payload_offset += 3 * sizeof(uint64_t) + input.id.size(); + + for (const BundleInput &input : inputs) { + result.entry_offsets.push_back(payload_offset); + AppendU64(result.bytes, payload_offset); + AppendU64(result.bytes, input.payload.size()); + AppendU64(result.bytes, input.id.size()); + result.bytes.insert(result.bytes.end(), input.id.bytes_begin(), + input.id.bytes_end()); + payload_offset += input.payload.size(); + } + + for (const BundleInput &input : inputs) + result.bytes.insert(result.bytes.end(), input.payload.begin(), + input.payload.end()); + return result; +} + +llvm::Expected<TestFile> MakeGPUELF(llvm::StringRef elf_flag) { + std::string yaml = R"( +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + OSAB... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/222362 _______________________________________________ lldb-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
