Skip to content

Commit 72f438c

Browse files
authored
Add support for extracting target images from fat offload binaries (#3548)
This patch consists of two parts 1) Change clang-offload-wrapper to create <address, size> pairs for all target images and add them to a .tgtimg section in the wrapper object. Contents of this section can later be used for locating and extracting target images from the fat offload binary using the new clang-offload-extract tool. 2) Add clang-offload-extract tool which allows extracting embedded target images using information from the .tgtimg section. Signed-off-by: Sergey Dmitriev <[email protected]>
1 parent 00253bf commit 72f438c

File tree

9 files changed

+285
-0
lines changed

9 files changed

+285
-0
lines changed

.github/CODEOWNERS

+1
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ llvm/lib/Support/Base64.cpp @mlychkov @AlexeySachkov @kbobrovs
9292
clang/tools/clang-offload-bundler/ @mlychkov @sndmitriev @AlexeySachkov
9393
clang/tools/clang-offload-wrapper/ @mlychkov @sndmitriev @AlexeySachkov @kbobrovs
9494
clang/tools/clang-offload-deps/ @sndmitriev @mlychkov @AlexeySachkov
95+
clang/tools/clang-offload-extract/ @sndmitriev @mlychkov @AlexeySachkov
9596

9697
# Explicit SIMD
9798
SYCLLowerIR/ @kbobrovs @DenisBakhvalov

clang/test/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ list(APPEND CLANG_TEST_DEPS
6666
clang-tblgen
6767
clang-offload-bundler
6868
clang-offload-deps
69+
clang-offload-extract
6970
clang-import-test
7071
clang-rename
7172
clang-refactor
+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// REQUIRES: x86-registered-target
2+
3+
//
4+
// Check help message.
5+
//
6+
// RUN: clang-offload-extract --help | FileCheck %s --check-prefix CHECK-HELP
7+
8+
// CHECK-HELP: OVERVIEW: A tool for extracting target images from the linked fat offload binary.
9+
// CHECK-HELP: USAGE: clang-offload-extract [options] <input file>
10+
// CHECK-HELP: OPTIONS:
11+
// CHECK-HELP: Generic Options:
12+
// CHECK-HELP: --help - Display available options (--help-hidden for more)
13+
// CHECK-HELP: --help-list - Display list of available options (--help-list-hidden for more)
14+
// CHECK-HELP: --version - Display the version of this program
15+
// CHECK-HELP: clang-offload-extract options:
16+
// CHECK-HELP: --output=<string> - Specifies prefix for the output file(s). Output file name
17+
// CHECK-HELP: is composed from this prefix and the sequential number
18+
// CHECK-HELP: of extracted image appended to the prefix.
19+
20+
//
21+
// Create fat offload binary with two embedded target images.
22+
//
23+
// RUN: echo 'Target image 1' > %t.bin0
24+
// RUN: echo 'Target image 2' > %t.bin1
25+
// RUN: clang-offload-wrapper -kind=openmp -target=tg0 %t.bin0 -kind=sycl -target=tg1 %t.bin1 -o %t.wrapped.bc
26+
// RUN: %clang %s %t.wrapped.bc -o %t.fat.bin
27+
28+
//
29+
// Extract target images.
30+
//
31+
// RUN: clang-offload-extract --output=%t.extracted %t.fat.bin | FileCheck %s --check-prefix CHECK-EXTRACT
32+
// CHECK-EXTRACT: Saving target image to
33+
// CHECK-EXTRACT: Saving target image to
34+
35+
//
36+
// Check that extracted contents match the original images.
37+
//
38+
// RUN: diff %t.extracted.0 %t.bin0
39+
// RUN: diff %t.extracted.1 %t.bin1
40+
41+
//
42+
// Some code so that we can build an offload executable from this file.
43+
//
44+
#ifdef _WIN32
45+
char __start_omp_offloading_entries = 1;
46+
char __stop_omp_offloading_entries = 1;
47+
#endif
48+
49+
void __tgt_register_lib(void *desc) {}
50+
void __tgt_unregister_lib(void *desc) {}
51+
52+
void __sycl_register_lib(void* desc) {}
53+
void __sycl_unregister_lib(void* desc) {}
54+
55+
int main(void) {
56+
return 0;
57+
}

clang/test/Driver/clang-offload-wrapper.c

+3
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@
128128
// CHECK-IR: [[DUMMY:@.+]] = hidden constant [0 x [[ENTTY]]] zeroinitializer, section "omp_offloading_entries"
129129

130130
// CHECK-IR: [[OMP_BIN:@.+]] = internal unnamed_addr constant [[OMP_BINTY:\[[0-9]+ x i8\]]] c"Content of device file3{{.+}}"
131+
// CHECK-IR: [[OMP_INFO:@.+]] = internal local_unnamed_addr constant [2 x i64] [i64 ptrtoint ([{{[0-9]+}} x i8]* [[OMP_BIN]] to i64), i64 24], section ".tgtimg", align 16
131132

132133
// CHECK-IR: [[OMP_IMAGES:@.+]] = internal unnamed_addr constant [1 x [[IMAGETY]]] [{{.+}} { i8* getelementptr inbounds ([[OMP_BINTY]], [[OMP_BINTY]]* [[OMP_BIN]], i64 0, i64 0), i8* getelementptr inbounds ([[OMP_BINTY]], [[OMP_BINTY]]* [[OMP_BIN]], i64 1, i64 0), [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }]
133134

@@ -137,11 +138,13 @@
137138
// CHECK-IR: [[SYCL_COMPILE_OPTS0:@.+]] = internal unnamed_addr constant [3 x i8] c"-g\00"
138139
// CHECK-IR: [[SYCL_LINK_OPTS0:@.+]] = internal unnamed_addr constant [21 x i8] c"-cl-denorms-are-zero\00"
139140
// CHECK-IR: [[SYCL_BIN0:@.+]] = internal unnamed_addr constant [[SYCL_BIN0TY:\[[0-9]+ x i8\]]] c"Content of device file1{{.+}}"
141+
// CHECK-IR: [[SYCL_INFO:@.+]] = internal local_unnamed_addr constant [2 x i64] [i64 ptrtoint ([{{[0-9]+}} x i8]* [[SYCL_BIN0]] to i64), i64 24], section ".tgtimg", align 16
140142

141143
// CHECK-IR: [[SYCL_TGT1:@.+]] = internal unnamed_addr constant [4 x i8] c"tg2\00"
142144
// CHECK-IR: [[SYCL_COMPILE_OPTS1:@.+]] = internal unnamed_addr constant [1 x i8] zeroinitializer
143145
// CHECK-IR: [[SYCL_LINK_OPTS1:@.+]] = internal unnamed_addr constant [1 x i8] zeroinitializer
144146
// CHECK-IR: [[SYCL_BIN1:@.+]] = internal unnamed_addr constant [[SYCL_BIN1TY:\[[0-9]+ x i8\]]] c"Content of device file2{{.+}}"
147+
// CHECK-IR: [[SYCL_INFO1:@.+]] = internal local_unnamed_addr constant [2 x i64] [i64 ptrtoint ([{{[0-9]+}} x i8]* [[SYCL_BIN1]] to i64), i64 24], section ".tgtimg", align 16
145148

146149
// CHECK-IR: [[SYCL_IMAGES:@.+]] = internal unnamed_addr constant [2 x [[SYCL_IMAGETY]]] [{{.*}} { i16 2, i8 4, i8 2, i8* getelementptr inbounds ([4 x i8], [4 x i8]* [[SYCL_TGT0]], i64 0, i64 0), i8* getelementptr inbounds ([3 x i8], [3 x i8]* [[SYCL_COMPILE_OPTS0]], i64 0, i64 0), i8* getelementptr inbounds ([21 x i8], [21 x i8]* [[SYCL_LINK_OPTS0]], i64 0, i64 0), i8* null, i8* null, i8* getelementptr inbounds ([[SYCL_BIN0TY]], [[SYCL_BIN0TY]]* [[SYCL_BIN0]], i64 0, i64 0), i8* getelementptr inbounds ([[SYCL_BIN0TY]], [[SYCL_BIN0TY]]* [[SYCL_BIN0]], i64 1, i64 0), [[ENTTY]]* null, [[ENTTY]]* null, [[PROPSETTY]]* null, [[PROPSETTY]]* null }, [[SYCL_IMAGETY]] { i16 2, i8 4, i8 1, i8* getelementptr inbounds ([4 x i8], [4 x i8]* [[SYCL_TGT1]], i64 0, i64 0), i8* getelementptr inbounds ([1 x i8], [1 x i8]* [[SYCL_COMPILE_OPTS1]], i64 0, i64 0), i8* getelementptr inbounds ([1 x i8], [1 x i8]* [[SYCL_LINK_OPTS1]], i64 0, i64 0), i8* null, i8* null, i8* getelementptr inbounds ([[SYCL_BIN1TY]], [[SYCL_BIN1TY]]* [[SYCL_BIN1]], i64 0, i64 0), i8* getelementptr inbounds ([[SYCL_BIN1TY]], [[SYCL_BIN1TY]]* [[SYCL_BIN1]], i64 1, i64 0), [[ENTTY]]* null, [[ENTTY]]* null, [[PROPSETTY]]* null, [[PROPSETTY]]* null }]
147150

clang/tools/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ add_clang_subdirectory(clang-offload-bundler)
1212
add_clang_subdirectory(clang-offload-deps)
1313
add_clang_subdirectory(clang-offload-wrapper)
1414
add_clang_subdirectory(clang-scan-deps)
15+
add_clang_subdirectory(clang-offload-extract)
1516

1617
add_clang_subdirectory(c-index-test)
1718

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
set(LLVM_LINK_COMPONENTS Core Object Support)
2+
3+
add_clang_tool(clang-offload-extract
4+
ClangOffloadExtract.cpp
5+
6+
DEPENDS
7+
intrinsics_gen
8+
)
9+
10+
set(CLANG_OFFLOAD_EXTRACT_LIB_DEPS
11+
clangBasic
12+
)
13+
14+
add_dependencies(clang clang-offload-extract)
15+
16+
clang_target_link_libraries(clang-offload-extract
17+
PRIVATE
18+
${CLANG_OFFLOAD_EXTRACT_LIB_DEPS}
19+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
//===-------- clang-offload-extract/ClangOffloadExtract.cpp ---------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
///
9+
/// \file
10+
/// Implementation of the clang-offload-extract tool which allows extracting
11+
/// target images from linked fat offload binaries. For locating target images
12+
/// in the binary it uses information from the .tgtimg section which is added to
13+
/// the image by the clang-offload-wrapper tool. This section contains <address,
14+
/// size> pairs for all embedded target images.
15+
///
16+
//===----------------------------------------------------------------------===//
17+
18+
#include "clang/Basic/Version.h"
19+
#include "llvm/ADT/ArrayRef.h"
20+
#include "llvm/Object/COFF.h"
21+
#include "llvm/Object/ELFObjectFile.h"
22+
#include "llvm/Object/ObjectFile.h"
23+
#include "llvm/Support/CommandLine.h"
24+
#include "llvm/Support/Errc.h"
25+
#include "llvm/Support/Signals.h"
26+
#include "llvm/Support/WithColor.h"
27+
#include "llvm/Support/raw_ostream.h"
28+
29+
#define IMAGE_INFO_SECTION_NAME ".tgtimg"
30+
31+
using namespace llvm;
32+
using namespace llvm::object;
33+
34+
static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
35+
36+
// Mark all our options with this category, everything else (except for -version
37+
// and -help) will be hidden.
38+
static cl::OptionCategory
39+
ClangOffloadExtractCategory("clang-offload-extract options");
40+
41+
static cl::opt<std::string> OutputPrefix(
42+
"output", cl::Required,
43+
cl::desc("Specifies prefix for the output file(s). Output file name\n"
44+
"is composed from this prefix and the sequential number\n"
45+
"of extracted image appended to the prefix."),
46+
cl::cat(ClangOffloadExtractCategory));
47+
48+
static cl::opt<std::string> Input(cl::Positional, cl::Required,
49+
cl::desc("<input file>"),
50+
cl::cat(ClangOffloadExtractCategory));
51+
52+
/// Path to the current binary.
53+
static std::string ToolPath;
54+
55+
static void reportError(Error E) {
56+
logAllUnhandledErrors(std::move(E), WithColor::error(errs(), ToolPath));
57+
}
58+
59+
int main(int argc, const char **argv) {
60+
sys::PrintStackTraceOnErrorSignal(argv[0]);
61+
ToolPath = argv[0];
62+
63+
cl::HideUnrelatedOptions(ClangOffloadExtractCategory);
64+
cl::SetVersionPrinter([](raw_ostream &OS) {
65+
OS << clang::getClangToolFullVersion("clang-offload-extract") << '\n';
66+
});
67+
cl::ParseCommandLineOptions(argc, argv,
68+
"A tool for extracting target images from the "
69+
"linked fat offload binary.");
70+
71+
if (Help) {
72+
cl::PrintHelpMessage();
73+
return 0;
74+
}
75+
76+
// Read input file. It should have one of the supported object file formats.
77+
Expected<OwningBinary<ObjectFile>> ObjectOrErr =
78+
ObjectFile::createObjectFile(Input);
79+
if (!ObjectOrErr) {
80+
reportError(ObjectOrErr.takeError());
81+
return 1;
82+
}
83+
84+
ObjectFile *Binary = ObjectOrErr->getBinary();
85+
86+
// Do we plan to support 32-bit offload binaries?
87+
if (!(isa<ELF64LEObjectFile>(Binary) || isa<COFFObjectFile>(Binary)) ||
88+
Binary->getBytesInAddress() != sizeof(void *)) {
89+
reportError(
90+
createStringError(errc::invalid_argument,
91+
"only 64-bit ELF or COFF inputs are supported"));
92+
return 1;
93+
}
94+
95+
unsigned FileNum = 0;
96+
97+
for (SectionRef Section : Binary->sections()) {
98+
// Look for the .tgtimg section in the binary.
99+
Expected<StringRef> NameOrErr = Section.getName();
100+
if (!NameOrErr) {
101+
reportError(NameOrErr.takeError());
102+
return 1;
103+
}
104+
if (*NameOrErr != IMAGE_INFO_SECTION_NAME)
105+
continue;
106+
107+
// This is the section we are looking for.
108+
Expected<StringRef> DataOrErr = Section.getContents();
109+
if (!DataOrErr) {
110+
reportError(DataOrErr.takeError());
111+
return 1;
112+
}
113+
114+
// This section contains concatenated <address, size> pairs describing
115+
// target images that are stored in the binary. Loop over these descriptors
116+
// and extract each target image.
117+
struct ImgInfoTy {
118+
uintptr_t Addr;
119+
uintptr_t Size;
120+
};
121+
122+
auto ImgInfo = makeArrayRef<ImgInfoTy>(
123+
reinterpret_cast<const ImgInfoTy *>(DataOrErr->data()),
124+
DataOrErr->size() / sizeof(ImgInfoTy));
125+
126+
for (auto &Img : ImgInfo) {
127+
// Find section which contains this image.
128+
// TODO: can use more efficient algorithm than linear search. For example
129+
// sections and images could be sorted by address then one pass performed
130+
// through both at the same time.
131+
auto ImgSec = find_if(Binary->sections(), [&Img](SectionRef Sec) {
132+
if (!Sec.isData())
133+
return false;
134+
if (Img.Addr < Sec.getAddress() ||
135+
Img.Addr + Img.Size > Sec.getAddress() + Sec.getSize())
136+
return false;
137+
return true;
138+
});
139+
if (ImgSec == Binary->section_end()) {
140+
reportError(createStringError(
141+
inconvertibleErrorCode(),
142+
"cannot find section containing <0x%lx, 0x%lx> target image",
143+
Img.Addr, Img.Size));
144+
return 1;
145+
}
146+
147+
Expected<StringRef> SecDataOrErr = ImgSec->getContents();
148+
if (!SecDataOrErr) {
149+
reportError(SecDataOrErr.takeError());
150+
return 1;
151+
}
152+
153+
// Output file name is composed from the name prefix provided by the user
154+
// and the image number which is appended to the prefix.
155+
std::string FileName = OutputPrefix + "." + std::to_string(FileNum++);
156+
157+
// Tell user that we are saving an image.
158+
outs() << "Saving target image to \"" << FileName << "\"\n";
159+
160+
// And write image data to the output.
161+
std::error_code EC;
162+
raw_fd_ostream OS(FileName, EC);
163+
if (EC) {
164+
reportError(createFileError(FileName, EC));
165+
return 1;
166+
}
167+
168+
OS << SecDataOrErr->substr(Img.Addr - ImgSec->getAddress(), Img.Size);
169+
if (OS.has_error()) {
170+
reportError(createFileError(FileName, OS.error()));
171+
return 1;
172+
}
173+
}
174+
175+
// Binary is not expected to have more than one .tgtimg section.
176+
break;
177+
}
178+
return 0;
179+
}

clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp

+22
Original file line numberDiff line numberDiff line change
@@ -958,6 +958,28 @@ class BinaryWrapper {
958958
} else
959959
ImagesInits.push_back(ConstantStruct::get(
960960
getDeviceImageTy(), Fbin.first, Fbin.second, EntriesB, EntriesE));
961+
962+
// Create an object that holds <address, size> pair for the device image
963+
// and put it into a .tgtimg section. This section can be used for finding
964+
// and extracting all device images from the fat binary after linking.
965+
Type *IntPtrTy = M.getDataLayout().getIntPtrType(C);
966+
auto *ImgInfoArr = ConstantArray::get(
967+
ArrayType::get(IntPtrTy, 2),
968+
{ConstantExpr::getPointerCast(Fbin.first, IntPtrTy),
969+
ConstantInt::get(IntPtrTy, Bin->getBufferSize())});
970+
auto *ImgInfoVar = new GlobalVariable(
971+
M, ImgInfoArr->getType(), /*isConstant*/ true,
972+
GlobalVariable::InternalLinkage, ImgInfoArr,
973+
Twine(OffloadKindTag) + Twine(ImgId) + Twine(".info"));
974+
ImgInfoVar->setAlignment(
975+
MaybeAlign(M.getDataLayout().getTypeStoreSize(IntPtrTy) * 2u));
976+
ImgInfoVar->setUnnamedAddr(GlobalValue::UnnamedAddr::Local);
977+
ImgInfoVar->setSection(".tgtimg");
978+
979+
// Add image info to the used list to force it to be emitted to the
980+
// object.
981+
appendToUsed(M, ImgInfoVar);
982+
961983
ImgId++;
962984
}
963985

sycl/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,7 @@ add_custom_target( sycl-toolchain
180180
clang-offload-wrapper
181181
clang-offload-bundler
182182
clang-offload-deps
183+
clang-offload-extract
183184
file-table-tform
184185
llc
185186
llvm-ar
@@ -226,6 +227,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS
226227
clang-offload-wrapper
227228
clang-offload-bundler
228229
clang-offload-deps
230+
clang-offload-extract
229231
file-table-tform
230232
level-zero-loader
231233
level-zero-headers

0 commit comments

Comments
 (0)