From 0dd267b73563acf1b6ee04eb5c9827ec5a56520f Mon Sep 17 00:00:00 2001 From: mingmingl Date: Tue, 13 Feb 2024 16:38:24 -0800 Subject: [PATCH 01/13] [TypeProf]Introduce value type in raw profiles --- compiler-rt/include/profile/InstrProfData.inc | 58 +++++- compiler-rt/lib/profile/InstrProfiling.h | 96 +++++++-- .../lib/profile/InstrProfilingBuffer.c | 58 +++++- .../lib/profile/InstrProfilingInternal.h | 4 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 30 ++- .../lib/profile/InstrProfilingPlatformLinux.c | 29 +++ .../lib/profile/InstrProfilingWriter.c | 37 +++- llvm/include/llvm/ProfileData/InstrProf.h | 172 +++++++++++++--- .../llvm/ProfileData/InstrProfData.inc | 42 +++- .../llvm/ProfileData/InstrProfWriter.h | 4 + llvm/lib/ProfileData/InstrProf.cpp | 188 +++++++++++++----- llvm/lib/ProfileData/InstrProfWriter.cpp | 59 +++++- .../InstrProfiling/coverage.ll | 8 +- .../Transforms/PGOProfile/comdat_internal.ll | 4 +- .../llvm-profdata/Inputs/c-general.profraw | Bin 2016 -> 2016 bytes .../thinlto_indirect_call_promotion.profraw | Bin 0 -> 528 bytes .../llvm-profdata/binary-ids-padding.test | 6 +- .../llvm-profdata/large-binary-id-size.test | 4 +- ...alformed-not-space-for-another-header.test | 6 +- .../malformed-num-counters-zero.test | 6 +- .../malformed-ptr-to-counter-array.test | 6 +- .../misaligned-binary-ids-size.test | 4 +- .../mismatched-raw-profile-header.test | 2 + .../tools/llvm-profdata/raw-32-bits-be.test | 11 +- .../tools/llvm-profdata/raw-32-bits-le.test | 10 +- .../tools/llvm-profdata/raw-64-bits-be.test | 10 +- .../tools/llvm-profdata/raw-64-bits-le.test | 10 +- .../tools/llvm-profdata/raw-two-profiles.test | 8 +- 28 files changed, 694 insertions(+), 178 deletions(-) create mode 100644 llvm/test/tools/llvm-profdata/Inputs/thinlto_indirect_call_promotion.profraw diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 25df899b3f361..f0bc2d960ce68 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -94,6 +94,26 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +/* For a virtual table object, record the name hash to associate profiled + * addresses with global variables, and record {starting address, size in bytes} + * to map the profiled virtual table (which usually have an offset from the + * starting address) back to a virtual table object. */ +#ifndef INSTR_PROF_VTABLE_DATA +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_VTABLE_DATA_DEFINED +#endif +INSTR_PROF_VTABLE_DATA( + const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), + VTablePointer, VTableAddr) +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + VTableSizeVal)) +#undef INSTR_PROF_VTABLE_DATA +/* INSTR_PROF_VTABLE_DATA end. */ /* This is an internal data structure used by value profiler. It * is defined here to allow serialization code sharing by LLVM @@ -145,6 +165,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -186,13 +208,28 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +/* For virtual table address profiling, the addresses of the virtual table + * (i.e., the address contained in objects pointing to a virtual table) are + * profiled. Note this may not be the address of the per C++ class virtual table + * object (i.e., there is an offset). + * + * The profiled addresses are stored in raw profile, together with the following + * two types of information. + * 1. The (beginning and ending) addresses of per C++ class virtual table objects. + * 2. The (compressed) virtual table object names. + * RawInstrProfReader converts profiled virtual table addresses to virtual table + * objects' MD5 hash. + */ +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., " + "there is an offset from this address to per C++ " + "class virtual table global variable.)") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") +VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -267,9 +304,9 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #undef COVMAP_HEADER /* COVMAP_HEADER end. */ - #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED + INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ INSTR_PROF_DATA_COFF, "__DATA,") @@ -282,12 +319,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vname, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ + INSTR_PROF_VNAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ + INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ + INSTR_PROF_VTAB_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") @@ -307,7 +350,6 @@ INSTR_PROF_SECT_ENTRY(IPSK_covname, \ #undef INSTR_PROF_SECT_ENTRY #endif - #ifdef INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_DATA_DEFINED @@ -479,7 +521,6 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #undef INSTR_PROF_VALUE_PROF_DATA #endif /* INSTR_PROF_VALUE_PROF_DATA */ - #ifdef INSTR_PROF_COMMON_API_IMPL #define INSTR_PROF_DATA_DEFINED #ifdef __cplusplus @@ -663,9 +704,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 10 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 12 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 6 @@ -703,10 +744,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, than WIN32 */ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts #define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_COVDATA_COMMON __llvm_covdata @@ -717,10 +760,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" #define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_VTAB_COFF ".lprfvt$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" /* Since cov data and cov names sections are not allocated, we don't need to @@ -938,3 +983,4 @@ InstrProfIsSingleValRange(uint64_t Value) { } #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ + diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 0123908336918..2ebb501a957bd 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -12,17 +12,6 @@ #include "InstrProfilingPort.h" #include -// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before -// including instr_prof_interface.h so the interface functions are -// declared correctly for the runtime. -// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header, -// because compiler-rt does not support profiling the profiling runtime itself. -#ifndef __LLVM_INSTR_PROFILE_GENERATE -#define __LLVM_INSTR_PROFILE_GENERATE -#endif -#include "profile/instr_prof_interface.h" -#undef __LLVM_INSTR_PROFILE_GENERATE - #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY #include "profile/InstrProfData.inc" @@ -49,6 +38,12 @@ typedef struct ValueProfNode { #include "profile/InstrProfData.inc" } ValueProfNode; +typedef void *IntPtrT; +typedef struct VTableProfData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} VTableProfData; + /*! * \brief Return 1 if profile counters are continuously synced to the raw * profile via an mmap(). This is in contrast to the default mode, in which @@ -103,14 +98,24 @@ const __llvm_profile_data *__llvm_profile_begin_data(void); const __llvm_profile_data *__llvm_profile_end_data(void); const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); +const char *__llvm_profile_begin_vtabnames(void); +const char *__llvm_profile_end_vtabnames(void); char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); char *__llvm_profile_begin_bitmap(void); char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +VTableProfData *__llvm_profile_begin_vtables(); +VTableProfData *__llvm_profile_end_vtables(); uint32_t *__llvm_profile_begin_orderfile(); +/*! + * \brief Clear profile counters to zero. + * + */ +void __llvm_profile_reset_counters(void); + /*! * \brief Merge profile data from buffer. * @@ -161,6 +166,50 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, int __llvm_profile_write_file(void); int __llvm_orderfile_write_file(void); +/*! + * \brief this is a wrapper interface to \c __llvm_profile_write_file. + * After this interface is invoked, an already dumped flag will be set + * so that profile won't be dumped again during program exit. + * Invocation of interface __llvm_profile_reset_counters will clear + * the flag. This interface is designed to be used to collect profile + * data from user selected hot regions. The use model is + * __llvm_profile_reset_counters(); + * ... hot region 1 + * __llvm_profile_dump(); + * .. some other code + * __llvm_profile_reset_counters(); + * ... hot region 2 + * __llvm_profile_dump(); + * + * It is expected that on-line profile merging is on with \c %m specifier + * used in profile filename . If merging is not turned on, user is expected + * to invoke __llvm_profile_set_filename to specify different profile names + * for different regions before dumping to avoid profile write clobbering. + */ +int __llvm_profile_dump(void); + +int __llvm_orderfile_dump(void); + +/*! + * \brief Set the filename for writing instrumentation data. + * + * Sets the filename to be used for subsequent calls to + * \a __llvm_profile_write_file(). + * + * \c Name is not copied, so it must remain valid. Passing NULL resets the + * filename logic to the default behaviour. + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the filename within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. + */ +void __llvm_profile_set_filename(const char *Name); /*! * \brief Set the FILE object for writing instrumentation data. Return 0 if set @@ -252,20 +301,31 @@ uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, /*! \brief Get the size of the profile name section in bytes. */ uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); -/* ! \brief Given the sizes of the data and counter information, return the - * number of padding bytes before and after the counters, and after the names, - * in the raw profile. +/*! \brief Get the number of virtual table profile data entries */ +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End); + +/*! \brief Get the size of virtual table profile data in bytes. */ +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End); + +/* ! \brief Given the sizes of the data and counter information, computes the + * number of padding bytes before and after the counter section, as well as the + * number of padding bytes after other setions in the raw profile. + * Returns -1 upon errors and 0 upon success. Output parameters should be used + * iff return value is 0. * * Note: When mmap() mode is disabled, no padding bytes before/after counters * are needed. However, in mmap() mode, the counter section in the raw profile * must be page-aligned: this API computes the number of padding bytes * needed to achieve that. */ -void __llvm_profile_get_padding_sizes_for_counters( +int __llvm_profile_get_padding_sizes_for_counters( uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmap, - uint64_t *PaddingBytesAfterNames); + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames); /*! * \brief Set the flag that profile data has been dumped to the file. diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index af52804b2b532..f31dc7d4e2111 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -70,6 +70,18 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End) { + intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; + return (EndI + sizeof(VTableProfData) - 1 - BeginI) / sizeof(VTableProfData); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End) { + return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData); +} COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) @@ -119,11 +131,13 @@ static int needsCounterPadding(void) { } COMPILER_RT_VISIBILITY -void __llvm_profile_get_padding_sizes_for_counters( +int __llvm_profile_get_padding_sizes_for_counters( uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, - uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters, - uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes, - uint64_t *PaddingBytesAfterNames) { + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) { + // Counter padding is needed only if continuous mode is enabled. if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; *PaddingBytesAfterCounters = @@ -131,9 +145,19 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesAfterBitmapBytes = __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); - return; + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = + __llvm_profile_get_num_padding_bytes(VTableSize); + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize); + return 0; } + // Value profiling not supported in continuous mode at profile-write time. + // Return -1 to alert the incompatibility. + if (VTableSize != 0 || VNameSize != 0) + return -1; + // In continuous mode, the file offsets for headers and for the start of // counter sections need to be page-aligned. *PaddingBytesBeforeCounters = @@ -142,6 +166,13 @@ void __llvm_profile_get_padding_sizes_for_counters( *PaddingBytesAfterBitmapBytes = calculateBytesNeededToPageAlign(NumBitmapBytes); *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); + // Set these two variables to zero to avoid uninitialized variables + // even if VTableSize and VNameSize are known to be zero. + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = 0; + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = 0; + return 0; } COMPILER_RT_VISIBILITY @@ -162,9 +193,11 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, + 0 /* VNameSize */, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */, + NULL /* PaddingbytesAfterVNames */); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + @@ -191,7 +224,10 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); - return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin, - NamesEnd, 0); + // Set virtual table arguments to NULL since they are not supported yet. + return lprofWriteDataImpl( + &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, + BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd, + NULL /* VTableBegin */, NULL /* VTableEnd */, NULL /* VNamesBegin */, + NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */); } diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 03ed67fcfa766..38159b668a1df 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -156,7 +156,9 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite); + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite); /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index b5850e99ee37d..53df1279ef961 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -41,9 +41,6 @@ uint64_t lprofGetLoadModuleSignature(void) { #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" -#elif defined(__clang__) -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wcast-qual" #endif /* Returns 1 if profile is not structurally compatible. */ @@ -107,6 +104,27 @@ static uintptr_t signextIfWin64(void *V) { #endif } +static uint64_t +getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { + // Skip names section, vtable profile data section and vtable names section + // for runtime profile merge. To merge runtime addresses from multiple + // profiles collected from the same instrumented binary, the binary should be + // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR + // disabled). + // In this set-up these three sections remain unchanged. + const uint64_t VTableSectionSize = + Header->NumVTables * sizeof(VTableProfData); + const uint64_t PaddingBytesAfterVTableSection = + __llvm_profile_get_num_padding_bytes(VTableSectionSize); + const uint64_t VNamesSize = Header->VNamesSize; + const uint64_t PaddingBytesAfterVNamesSize = + __llvm_profile_get_num_padding_bytes(VNamesSize); + return Header->NamesSize + + __llvm_profile_get_num_padding_bytes(Header->NamesSize) + + VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize + + PaddingBytesAfterVNamesSize; +} + COMPILER_RT_VISIBILITY int __llvm_profile_merge_from_buffer(const char *ProfileData, uint64_t ProfileSize) { @@ -136,9 +154,9 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, Header->NumCounters * __llvm_profile_counter_entry_size(); SrcBitmapStart = SrcCountersEnd; SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; + SrcValueProfDataStart = - SrcNameStart + Header->NamesSize + - __llvm_profile_get_num_padding_bytes(Header->NamesSize); + SrcNameStart + getDistanceFromCounterToValueProf(Header); if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; @@ -237,6 +255,4 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, #ifdef __GNUC__ #pragma GCC diagnostic pop -#elif defined(__clang__) -#pragma clang diagnostic pop #endif diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index 19266ab6c6fb8..dabe0e900465e 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -20,12 +20,25 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" +#if defined(__FreeBSD__) && !defined(ElfW) +/* + * FreeBSD's elf.h and link.h headers do not define the ElfW(type) macro yet. + * If this is added to all supported FreeBSD versions in the future, this + * compatibility macro can be removed. + */ +#define ElfW(type) __ElfN(type) +#endif + #define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON) +#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON) +#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON) #define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) #define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) @@ -41,6 +54,10 @@ extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; @@ -63,6 +80,18 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &PROF_NAME_STOP; } +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) { + return &PROF_VNAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) { + return &PROF_VNAME_STOP; +} +COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_begin_vtables(void) { + return &PROF_VTABLE_START; +} +COMPILER_RT_VISIBILITY VTableProfData *__llvm_profile_end_vtables(void) { + return &PROF_VTABLE_STOP; +} COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return &PROF_CNTS_START; } diff --git a/compiler-rt/lib/profile/InstrProfilingWriter.c b/compiler-rt/lib/profile/InstrProfilingWriter.c index 4d767d1385148..8816a71155511 100644 --- a/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -250,9 +250,14 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, - NamesBegin, NamesEnd, SkipNameDataWrite); + NamesBegin, NamesEnd, VTableBegin, VTableEnd, + VNamesBegin, VNamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int @@ -261,7 +266,9 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite) { + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite) { /* Calculate size of sections. */ const uint64_t DataSectionSize = __llvm_profile_get_data_size(DataBegin, DataEnd); @@ -273,6 +280,12 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const uint64_t NumBitmapBytes = __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); + const uint64_t NumVTables = + __llvm_profile_get_num_vtable(VTableBegin, VTableEnd); + const uint64_t VTableSectionSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNamesSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Create the header. */ __llvm_profile_header Header; @@ -280,11 +293,15 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; - __llvm_profile_get_padding_sizes_for_counters( - DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, - &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, - &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames); + PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + if (__llvm_profile_get_padding_sizes_for_counters( + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, + VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames) == -1) + return -1; { /* Initialize header structure. */ @@ -323,7 +340,11 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, - {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; + {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}, + {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1}, + {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}}; if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index a928ba6961f36..16073521a976d 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -89,6 +89,9 @@ inline StringRef getInstrProfValueProfMemOpFuncName() { /// Return the name prefix of variables containing instrumented function names. inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } +/// Return the name prefix of variables containing virtual table profile data. +inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; } + /// Return the name prefix of variables containing per-function control data. inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } @@ -110,6 +113,8 @@ inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; } +inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; } + /// Return the name of a covarage mapping variable (internal linkage) /// for each instrumented source module. Such variables are allocated /// in the __llvm_covmap section. @@ -195,7 +200,7 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO = false); /// \return the filename and the function name parsed from the output of /// \c getIRPGOFuncName() -std::pair getParsedIRPGOName(StringRef IRPGOName); +std::pair getParsedIRPGOName(StringRef IRPGOFuncName); /// Return the name of the global variable used to store a function /// name in PGO instrumentation. \c FuncName is the IRPGO function name @@ -246,6 +251,9 @@ Error collectGlobalObjectNameStrings(ArrayRef NameStrs, Error collectPGOFuncNameStrings(ArrayRef NameVars, std::string &Result, bool doCompression = true); +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool doCompression); + /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being /// set in IR PGO compilation. bool isIRPGOFlagSet(const Module *M); @@ -288,6 +296,8 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } /// Return the PGOFuncName meta data associated with a function. MDNode *getPGOFuncNameMetadata(const Function &F); +std::string getPGOName(const GlobalVariable &V, bool InLTO = false); + /// Create the PGOFuncName meta data if PGOFuncName is different from /// function's raw name. This should only apply to internal linkage functions /// declared by users only. @@ -295,7 +305,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); /// Check if we can use Comdat for profile variables. This will eliminate /// the duplicated profile variables for Comdat functions. -bool needsComdatForCounter(const Function &F, const Module &M); +bool needsComdatForCounter(const GlobalValue &GV, const Module &M); /// An enum describing the attributes of an instrumented profile. enum class InstrProfKind { @@ -429,37 +439,42 @@ uint64_t ComputeHash(StringRef K); class InstrProfSymtab { public: using AddrHashMap = std::vector>; + using RangeHashMap = + std::vector, uint64_t>>; private: StringRef Data; uint64_t Address = 0; - // Unique name strings. + // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's + // going to be sorted) has unique MD5 keys in the first place. StringSet<> NameTab; + // Records the unique virtual table names. This is used by InstrProfWriter to + // write out an on-disk chained hash table of virtual table names. + // InstrProfWriter stores per function profile data (keyed by function names) + // so it doesn't use a StringSet for function names. + StringSet<> VTableNames; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; + // A map from MD5 keys to virtual table definitions. Only populated when + // building the Symtab from a module. + std::vector> MD5VTableMap; // A map from MD5 keys to function define. We only populate this map // when build the Symtab from a Module. std::vector> MD5FuncMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; + // A map from virtual table runtime address to function name MD5 hash. + // This map is only populated and used by raw instr profile reader. + // This is a different map from 'AddrToMD5Map' for readability and + // debuggability. + RangeHashMap VTableAddrRangeToMD5Map; bool Sorted = false; static StringRef getExternalSymbol() { return "** External Symbol **"; } - // Returns the canonial name of the given PGOName. In a canonical name, all - // suffixes that begins with "." except ".__uniq." are stripped. - // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. - static StringRef getCanonicalName(StringRef PGOName); - - // Add the function into the symbol table, by creating the following - // map entries: - // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical - // name is different from pgo name - // - In MD5NameMap: for name in name-set - // - In MD5FuncMap: for name in name-set Error addFuncWithName(Function &F, StringRef PGOFuncName); // If the symtab is created by a series of calls to \c addFuncName, \c @@ -481,9 +496,19 @@ class InstrProfSymtab { /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. - /// This method is a wrapper to \c readPGOFuncNameStrings method. + /// This method is a wrapper to \c readAndDecodeStrings method. Error create(StringRef NameStrings); + /// \c FuncNameStrings is a string composed of one or more encoded function + /// name strings, and \c VTableNameStrings composes of one or more encoded + /// vtable names. This function is a wrapper to \c readAndDecodeStrings + /// method. + Error create(StringRef FuncNameStrings, StringRef VTableNameStrings); + + /// Initialize 'this' with the set of vtable names encoded in + /// \c CompressedVTableNames. + Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames); + /// This interface is used by reader of CoverageMapping test /// format. inline Error create(StringRef D, uint64_t BaseAddr); @@ -496,32 +521,70 @@ class InstrProfSymtab { /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. - template Error create(const NameIterRange &IterRange); - - /// Update the symtab by adding \p FuncName to the table. This interface - /// is used by the raw and text profile readers. - Error addFuncName(StringRef FuncName) { - if (FuncName.empty()) + template + Error create(const NameIterRange &IterRange); + + /// Create InstrProfSymtab from a set of function names and vtable + /// names iteratable from \p IterRange. This interface is used by + /// IndexedProfReader. + template + Error create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange); + + Error addSymbolName(StringRef SymbolName) { + if (SymbolName.empty()) return make_error(instrprof_error::malformed, - "function name is empty"); - auto Ins = NameTab.insert(FuncName); + "symbol name is empty"); + + // Insert into NameTab so that MD5NameMap (a vector that will be sorted) + // won't have duplicated entries in the first place. + auto Ins = NameTab.insert(SymbolName); if (Ins.second) { MD5NameMap.push_back(std::make_pair( - IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); + IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey())); Sorted = false; } return Error::success(); } + /// The method name is kept since there are many callers. + /// It just forwards to 'addSymbolName'. + Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); } + + /// Adds VTableName as a known symbol, and inserts it to a map that + /// tracks all vtable names. + Error addVTableName(StringRef VTableName) { + if (Error E = addSymbolName(VTableName)) + return E; + + // Record VTableName. InstrProfWriter uses this map. The comment around + // class member explains why. + VTableNames.insert(VTableName); + return Error::success(); + } + + const StringSet<> &getVTableNames() const { return VTableNames; } + /// Map a function address to its name's MD5 hash. This interface /// is only used by the raw profiler reader. void mapAddress(uint64_t Addr, uint64_t MD5Val) { AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); } + /// Map the address range (i.e., [start_address, end_address]) of a variable + /// to its names' MD5 hash. This interface is only used by the raw profile + /// reader. + void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) { + VTableAddrRangeToMD5Map.push_back( + std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val)); + } + /// Return a function's hash, or 0, if the function isn't in this SymTab. uint64_t getFunctionHashFromAddress(uint64_t Address); + /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab. + uint64_t getVTableHashFromAddress(uint64_t Address); + /// Return function's PGO name from the function name's symbol /// address in the object file. If an error occurs, return /// an empty string. @@ -543,6 +606,8 @@ class InstrProfSymtab { /// Return function from the name's md5 hash. Return nullptr if not found. inline Function *getFunction(uint64_t FuncMD5Hash); + // Return vtable from the name's MD5 hash. Return nullptr if not found. + inline GlobalVariable *getGlobalVariable(uint64_t GlobalVariableMD5Hash); /// Return the name section data. inline StringRef getNameData() const { return Data; } @@ -567,6 +632,23 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) { return Error::success(); } +template +Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, + const VTableNameIterRange &VTableIterRange) { + for (auto Name : FuncIterRange) + if (Error E = addFuncName(Name)) + return E; + + for (auto VTableName : VTableIterRange) { + if (Error E = addVTableName(VTableName)) { + return E; + } + } + + finalizeSymtab(); + return Error::success(); +} + void InstrProfSymtab::finalizeSymtab() { if (Sorted) return; @@ -575,6 +657,13 @@ void InstrProfSymtab::finalizeSymtab() { llvm::sort(AddrToMD5Map, less_first()); AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), AddrToMD5Map.end()); + // VTable object address ranges should not overlap; so sort by either + // beginning address or end address is fine. + llvm::sort(VTableAddrRangeToMD5Map, less_first()); + // std::unique uses == operator for std::pair. + VTableAddrRangeToMD5Map.erase(std::unique(VTableAddrRangeToMD5Map.begin(), + VTableAddrRangeToMD5Map.end()), + VTableAddrRangeToMD5Map.end()); Sorted = true; } @@ -605,6 +694,19 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { return nullptr; } +GlobalVariable * +InstrProfSymtab::getGlobalVariable(uint64_t GlobalVariableMD5Hash) { + finalizeSymtab(); + auto Result = + llvm::lower_bound(MD5VTableMap, GlobalVariableMD5Hash, + [](const std::pair &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + + if (Result != MD5VTableMap.end() && Result->first == GlobalVariableMD5Hash) + return Result->second; + return nullptr; +} + // To store the sums of profile count values, or the percentage of // the sums of the total count values. struct CountSumOrPercent { @@ -831,6 +933,7 @@ struct InstrProfRecord { struct ValueProfData { std::vector IndirectCallSites; std::vector MemOPSizes; + std::vector VTableTargets; }; std::unique_ptr ValueData; @@ -853,6 +956,8 @@ struct InstrProfRecord { return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; + case IPVK_VTableTarget: + return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } @@ -867,6 +972,8 @@ struct InstrProfRecord { return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; + case IPVK_VTableTarget: + return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } @@ -1036,7 +1143,9 @@ enum ProfVersion { Version10 = 10, // An additional field is used for bitmap bytes. Version11 = 11, - // The current version is 11. + // VTable profiling, + Version12 = 12, + // The current version is 12. CurrentVersion = INSTR_PROF_INDEX_VERSION }; const uint64_t Version = ProfVersion::CurrentVersion; @@ -1046,8 +1155,7 @@ const HashT HashType = HashT::MD5; inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } // This structure defines the file header of the LLVM profile -// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst -// as appropriate when updating the indexed profile format. +// data file in indexed-format. struct Header { uint64_t Magic; uint64_t Version; @@ -1057,6 +1165,7 @@ struct Header { uint64_t MemProfOffset; uint64_t BinaryIdOffset; uint64_t TemporalProfTracesOffset; + uint64_t VTableNamesOffset; // Organize virtual table names. // New fields should only be added at the end to ensure that the size // computation is correct. The methods below need to be updated to ensure that // the new field is read correctly. @@ -1193,8 +1302,13 @@ template <> inline uint64_t getMagic() { // It should also match the synthesized type in // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. template struct alignas(8) ProfileData { - #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; - #include "llvm/ProfileData/InstrProfData.inc" +#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" +}; + +template struct alignas(8) VTableProfileData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name; +#include "llvm/ProfileData/InstrProfData.inc" }; // File header structure of the LLVM profile data in raw format. diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 25df899b3f361..f6410d6696f49 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -94,6 +94,22 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +#ifndef INSTR_PROF_VTABLE_DATA +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_VTABLE_DATA_DEFINED +#endif +INSTR_PROF_VTABLE_DATA( + const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), + VTablePointer, VTableAddr) +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), + VTableSizeVal)) +#undef INSTR_PROF_VTABLE_DATA +/* INSTR_PROF_VTABLE_DATA end. */ /* This is an internal data structure used by value profiler. It * is defined here to allow serialization code sharing by LLVM @@ -123,8 +139,6 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ /* INSTR_PROF_RAW_HEADER start */ /* Definition of member fields of the raw profile header data structure. */ -/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating - raw profile format. */ #ifndef INSTR_PROF_RAW_HEADER #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) #else @@ -145,6 +159,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -186,13 +202,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") +VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -267,7 +284,6 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #undef COVMAP_HEADER /* COVMAP_HEADER end. */ - #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ @@ -282,12 +298,18 @@ INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vname, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ + INSTR_PROF_VNAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ + INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ + INSTR_PROF_VTAB_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") @@ -307,7 +329,6 @@ INSTR_PROF_SECT_ENTRY(IPSK_covname, \ #undef INSTR_PROF_SECT_ENTRY #endif - #ifdef INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_DATA_DEFINED @@ -347,7 +368,7 @@ typedef struct ValueProfRecord { /*! * Return the number of value sites. */ - uint32_t getNumValueSites() const { return NumValueSites; } + uint32_t getNumValueSites() const { return NumValueSites; } /*! * Read data from this record and save it to Record. */ @@ -479,7 +500,6 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #undef INSTR_PROF_VALUE_PROF_DATA #endif /* INSTR_PROF_VALUE_PROF_DATA */ - #ifdef INSTR_PROF_COMMON_API_IMPL #define INSTR_PROF_DATA_DEFINED #ifdef __cplusplus @@ -663,9 +683,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 9 +#define INSTR_PROF_RAW_VERSION 10 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 11 +#define INSTR_PROF_INDEX_VERSION 12 /* Coverage mapping format version (start from 0). */ #define INSTR_PROF_COVMAP_VERSION 6 @@ -703,10 +723,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, than WIN32 */ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_VNAME_COMMON __llvm_prf_vtabnames #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts #define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab #define INSTR_PROF_COVMAP_COMMON __llvm_covmap #define INSTR_PROF_COVFUN_COMMON __llvm_covfun #define INSTR_PROF_COVDATA_COMMON __llvm_covdata @@ -717,10 +739,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfvn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" #define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_VTAB_COFF ".lprfvt$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" #define INSTR_PROF_COVFUN_COFF ".lcovfun$M" /* Since cov data and cov names sections are not allocated, we don't need to diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index 047b14f223bd9..049fa36bb53f5 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -63,6 +63,9 @@ class InstrProfWriter { // List of binary ids. std::vector BinaryIds; + // Read the vtable names from raw instr profile reader. + StringSet<> VTableNames; + // An enum describing the attributes of the profile. InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. @@ -84,6 +87,7 @@ class InstrProfWriter { void addRecord(NamedInstrProfRecord &&I, function_ref Warn) { addRecord(std::move(I), 1, Warn); } + void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); } /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the /// total number of temporal profiling traces the source has seen. diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 2eeeff987399d..96a1df04edd47 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -14,6 +14,7 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" @@ -26,6 +27,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" @@ -219,6 +221,12 @@ cl::opt DoInstrProfNameCompression( "enable-name-compression", cl::desc("Enable name/filename string compression"), cl::init(true)); +cl::opt EnableVTableValueProfiling( + "enable-vtable-value-profiling", cl::init(false), + cl::desc("If true, the virtual table address will be instrumented to know " + "the types of a C++ pointer. The information is used in indirect " + "call promotion to do selective vtable-based comparison.")); + std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { @@ -295,20 +303,29 @@ static StringRef getStrippedSourceFileName(const GlobalObject &GO) { return FileName; } -// The PGO name has the format [;] where ; is -// provided if linkage is local and is used to discriminate possibly identical -// mangled names. ";" is used because it is unlikely to be found in either -// or . +// The PGO name has the format [;] where ; is +// provided if linkage is local and is the mangled function +// name. The filepath is used to discriminate possibly identical function names. +// ; is used because it is unlikely to be found in either or +// . // // Older compilers used getPGOFuncName() which has the format -// [:]. This caused trouble for Objective-C functions -// which commonly have :'s in their names. We still need to compute this name to -// lookup functions from profiles built by older compilers. +// [:]. is used to discriminate between +// possibly identical function names when linkage is local and +// simply comes from F.getName(). This caused trouble for Objective-C functions +// which commonly have :'s in their names. Also, since is not +// mangled, they cannot be passed to Mach-O linkers via -order_file. We still +// need to compute this name to lookup functions from profiles built by older +// compilers. static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { - return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName); + SmallString<64> Name; + // FIXME: Mangler's handling is kept outside of `getGlobalIdentifier` for now. + // For more details please check issue #74565. + Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); + return GlobalValue::getGlobalIdentifier(Name, Linkage, FileName); } static std::optional lookupPGONameFromMetadata(MDNode *MD) { @@ -378,12 +395,19 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } -// See getIRPGOObjectName() for a discription of the format. -std::pair getParsedIRPGOName(StringRef IRPGOName) { - auto [FileName, MangledName] = IRPGOName.split(kGlobalIdentifierDelimiter); - if (MangledName.empty()) - return std::make_pair(StringRef(), IRPGOName); - return std::make_pair(FileName, MangledName); +std::string getPGOName(const GlobalVariable &V, bool InLTO) { + // PGONameMetadata should be set by compiler at profile use time + // and read by symtab creation to look up symbols corresponding to + // a MD5 hash. + return getIRPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */); +} + +// See getIRPGOFuncName() for a discription of the format. +std::pair getParsedIRPGOName(StringRef IRPGOFuncName) { + auto [FileName, FuncName] = IRPGOFuncName.split(';'); + if (FuncName.empty()) + return std::make_pair(StringRef(), IRPGOFuncName); + return std::make_pair(FileName, FuncName); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -459,6 +483,17 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) return E; } + + SmallVector Types; + for (GlobalVariable &G : M.globals()) { + if (!G.hasName()) + continue; + Types.clear(); + G.getMetadata(LLVMContext::MD_type, Types); + if (!Types.empty()) { + MD5VTableMap.emplace_back(G.getGUID(), &G); + } + } Sorted = false; finalizeSymtab(); return Error::success(); @@ -517,49 +552,79 @@ Error InstrProfSymtab::create(StringRef NameStrings) { std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1)); } -StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { +Error InstrProfSymtab::create(StringRef FuncNameStrings, + StringRef VTableNameStrings) { + if (Error E = readAndDecodeStrings(FuncNameStrings, + std::bind(&InstrProfSymtab::addFuncName, + this, std::placeholders::_1))) + return E; + + return readAndDecodeStrings( + VTableNameStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + +Error InstrProfSymtab::initVTableNamesFromCompressedStrings( + StringRef CompressedVTableStrings) { + return readAndDecodeStrings( + CompressedVTableStrings, + std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); +} + +Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { + if (Error E = addFuncName(PGOFuncName)) + return E; + MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); // In ThinLTO, local function may have been promoted to global and have // suffix ".llvm." added to the function name. We need to add the // stripped function name to the symbol table so that we can find a match // from profile. // - // ".__uniq." suffix is used to differentiate internal linkage functions in - // different modules and should be kept. This is the only suffix with the - // pattern ".xxx" which is kept before matching, other suffixes similar as - // ".llvm." will be stripped. + // We may have other suffixes similar as ".llvm." which are needed to + // be stripped before the matching, but ".__uniq." suffix which is used + // to differentiate internal linkage functions in different modules + // should be kept. Now this is the only suffix with the pattern ".xxx" + // which is kept before matching. const std::string UniqSuffix = ".__uniq."; - size_t pos = PGOName.find(UniqSuffix); - if (pos != StringRef::npos) + auto pos = PGOFuncName.find(UniqSuffix); + // Search '.' after ".__uniq." if ".__uniq." exists, otherwise + // search '.' from the beginning. + if (pos != std::string::npos) pos += UniqSuffix.length(); else pos = 0; - - // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from - // the beginning. - pos = PGOName.find('.', pos); - if (pos != StringRef::npos && pos != 0) - return PGOName.substr(0, pos); - - return PGOName; -} - -Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { - auto mapName = [&](StringRef Name) -> Error { - if (Error E = addFuncName(Name)) + pos = PGOFuncName.find('.', pos); + if (pos != std::string::npos && pos != 0) { + StringRef OtherFuncName = PGOFuncName.substr(0, pos); + if (Error E = addFuncName(OtherFuncName)) return E; - MD5FuncMap.emplace_back(Function::getGUID(Name), &F); - return Error::success(); - }; - if (Error E = mapName(PGOFuncName)) - return E; - - StringRef CanonicalFuncName = getCanonicalName(PGOFuncName); - if (CanonicalFuncName != PGOFuncName) - return mapName(CanonicalFuncName); - + MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); + } return Error::success(); } +uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) { + finalizeSymtab(); + auto It = lower_bound( + VTableAddrRangeToMD5Map, Address, + [](std::pair, uint64_t> VTableRangeAddr, + uint64_t Addr) { + // Find the first address range of which end address is larger than + // `Addr`. Smaller-than-or-equal-to is used because the profiled address + // within a vtable should be [start-address, end-address). + return VTableRangeAddr.first.second <= Addr; + }); + + // Returns the MD5 hash if Address is within the address range of an entry. + if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) { + return It->second; + } + // The virtual table address collected from value profiler could be defined + // in another module that is not instrumented. Force the value to be 0 in + // this case. + return 0; +} + uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { finalizeSymtab(); auto It = partition_point(AddrToMD5Map, [=](std::pair A) { @@ -636,6 +701,17 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, NameStrs, compression::zlib::isAvailable() && doCompression, Result); } +Error collectVTableStrings(ArrayRef VTables, + std::string &Result, bool doCompression) { + std::vector VTableNameStrs; + for (auto *VTable : VTables) { + VTableNameStrs.push_back(getPGOName(*VTable)); + } + return collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable() && doCompression, + Result); +} + void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { uint64_t FuncSum = 0; Sum.NumEntries += Counts.size(); @@ -898,6 +974,9 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, if (ValueKind == IPVK_IndirectCallTarget) return SymTab->getFunctionHashFromAddress(Value); + if (ValueKind == IPVK_VTableTarget) + return SymTab->getVTableHashFromAddress(Value); + return Value; } @@ -1288,8 +1367,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { F.setMetadata(getPGOFuncNameMetadataName(), N); } -bool needsComdatForCounter(const Function &F, const Module &M) { - if (F.hasComdat()) +bool needsComdatForCounter(const GlobalValue &GV, const Module &M) { + if (GV.hasComdat()) return true; if (!Triple(M.getTargetTriple()).supportsCOMDAT()) @@ -1305,7 +1384,7 @@ bool needsComdatForCounter(const Function &F, const Module &M) { // available_externally functions will end up being duplicated in raw profile // data. This can result in distorted profile as the counts of those dups // will be accumulated by the profile merger. - GlobalValue::LinkageTypes Linkage = F.getLinkage(); + GlobalValue::LinkageTypes Linkage = GV.getLinkage(); if (Linkage != GlobalValue::ExternalWeakLinkage && Linkage != GlobalValue::AvailableExternallyLinkage) return false; @@ -1461,7 +1540,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) continue; - char ProfileKindName[20]; + char ProfileKindName[20] = {0}; switch (I) { case IPVK_IndirectCallTarget: strncpy(ProfileKindName, "IndirectCall", 19); @@ -1469,6 +1548,9 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { case IPVK_MemOPSize: strncpy(ProfileKindName, "MemOP", 19); break; + case IPVK_VTableTarget: + strncpy(ProfileKindName, "VTable", 19); + break; default: snprintf(ProfileKindName, 19, "VP[%d]", I); break; @@ -1533,9 +1615,12 @@ Expected
Header::readFromBuffer(const unsigned char *Buffer) { // When a new field is added in the header add a case statement here to // populate it. static_assert( - IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + IndexedInstrProf::ProfVersion::CurrentVersion == Version12, "Please update the reading code below if a new field has been added, " "if not add a case statement to fall through to the latest version."); + case 12ull: + H.VTableNamesOffset = read(Buffer, offsetOf(&Header::VTableNamesOffset)); + [[fallthrough]]; case 11ull: [[fallthrough]]; case 10ull: @@ -1561,10 +1646,13 @@ size_t Header::size() const { // When a new field is added to the header add a case statement here to // compute the size as offset of the new field + size of the new field. This // relies on the field being added to the end of the list. - static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version11, + static_assert(IndexedInstrProf::ProfVersion::CurrentVersion == Version12, "Please update the size computation below if a new field has " "been added to the header, if not add a case statement to " "fall through to the latest version."); + case 12ull: + return offsetOf(&Header::VTableNamesOffset) + + sizeof(Header::VTableNamesOffset); case 11ull: [[fallthrough]]; case 10ull: diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index d65f8fe50313d..7592c0ffd3272 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -19,6 +19,7 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/MemProf.h" #include "llvm/ProfileData/ProfileCommon.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -455,12 +456,13 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { Header.MemProfOffset = 0; Header.BinaryIdOffset = 0; Header.TemporalProfTracesOffset = 0; + Header.VTableNamesOffset = 0; int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); // Only write out all the fields except 'HashOffset', 'MemProfOffset', - // 'BinaryIdOffset' and `TemporalProfTracesOffset`. We need to remember the - // offset of these fields to allow back patching later. - for (int I = 0; I < N - 4; I++) + // 'BinaryIdOffset', `TemporalProfTracesOffset` and `VTableNamesOffset`. We + // need to remember the offset of these fields to allow back patching later. + for (int I = 0; I < N - 5; I++) OS.write(reinterpret_cast(&Header)[I]); // Save the location of Header.HashOffset field in \c OS. @@ -484,6 +486,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { uint64_t TemporalProfTracesOffset = OS.tell(); OS.write(0); + uint64_t VTableNamesOffset = OS.tell(); + OS.write(0); + // Reserve space to write profile summary data. uint32_t NumEntries = ProfileSummaryBuilder::DefaultCutoffs.size(); uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries); @@ -604,6 +609,43 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { OS.writeByte(0); } + // if version >= the version with vtable profile metadata. + uint64_t VTableNamesSectionStart = 0; + if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) { + VTableNamesSectionStart = OS.tell(); + + std::string CompressedVTableNames; + + std::vector VTableNameStrs; + for (const auto &VTableName : VTableNames.keys()) { + VTableNameStrs.push_back(VTableName.str()); + } + + if (!VTableNameStrs.empty()) { + if (Error E = collectGlobalObjectNameStrings( + VTableNameStrs, compression::zlib::isAvailable(), + CompressedVTableNames)) + return E; + } + + uint64_t CompressedStringLen = CompressedVTableNames.length(); + + // Record the length of compressed string. + OS.write(CompressedStringLen); + + // Write the chars in compressed strings. + for (auto &c : CompressedVTableNames) + OS.writeByte(static_cast(c)); + + // Pad up to a multiple of 8. + // InstrProfReader could read bytes according to 'CompressedStringLen'. + uint64_t PaddedLength = alignTo(CompressedStringLen, 8); + + for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { + OS.writeByte(0); + } + } + uint64_t TemporalProfTracesSectionStart = 0; if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) { TemporalProfTracesSectionStart = OS.tell(); @@ -647,6 +689,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { // Patch the Header.TemporalProfTracesOffset (=0 for profiles without // traces). {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1}, + {VTableNamesOffset, &VTableNamesSectionStart, 1}, // Patch the summary data. {SummaryOffset, reinterpret_cast(TheSummary.get()), (int)(SummarySize / sizeof(uint64_t))}, @@ -699,7 +742,8 @@ Error InstrProfWriter::validateRecord(const InstrProfRecord &Func) { std::unique_ptr VD = Func.getValueForSite(VK, S); DenseSet SeenValues; for (uint32_t I = 0; I < ND; I++) - if ((VK != IPVK_IndirectCallTarget) && !SeenValues.insert(VD[I].Value).second) + if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) && + !SeenValues.insert(VD[I].Value).second) return make_error(instrprof_error::invalid_prof); } } @@ -747,7 +791,7 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash, OS << ND << "\n"; std::unique_ptr VD = Func.getValueForSite(VK, S); for (uint32_t I = 0; I < ND; I++) { - if (VK == IPVK_IndirectCallTarget) + if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget) OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":" << VD[I].Count << "\n"; else @@ -786,6 +830,11 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { } } + for (const auto &VTableName : VTableNames) { + if (Error E = Symtab.addVTableName(VTableName.getKey())) + return E; + } + if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) writeTextTemporalProfTraceData(OS, Symtab); diff --git a/llvm/test/Instrumentation/InstrProfiling/coverage.ll b/llvm/test/Instrumentation/InstrProfiling/coverage.ll index bbf895ea4b34e..08cbcaa962b76 100644 --- a/llvm/test/Instrumentation/InstrProfiling/coverage.ll +++ b/llvm/test/Instrumentation/InstrProfiling/coverage.ll @@ -5,12 +5,12 @@ target triple = "aarch64-unknown-linux-gnu" @__profn_foo = private constant [3 x i8] c"foo" ; CHECK: @__profc_foo = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 -; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) -; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), +; CHECK: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_foo to i64) +; BINARY: @__profd_foo = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_foo to i64), @__profn_bar = private constant [3 x i8] c"bar" ; CHECK: @__profc_bar = private global [1 x i8] c"\FF", section "__llvm_prf_cnts", comdat, align 1 -; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) -; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), +; CHECK: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 sub (i64 ptrtoint (ptr @__profc_bar to i64) +; BINARY: @__profd_bar = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 {{.*}}, i64 ptrtoint (ptr @__profc_bar to i64), ; CHECK: @__llvm_prf_nm = {{.*}} section "__llvm_prf_names" ; BINARY: @__llvm_prf_nm ={{.*}} section "__llvm_covnames" diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll index 8c6942c0f527b..1bad0db1b4762 100644 --- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll +++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll @@ -13,9 +13,9 @@ $foo = comdat any ; CHECK: @__llvm_profile_raw_version = hidden constant i64 {{[0-9]+}}, comdat ; CHECK-NOT: __profn__stdin__foo ; CHECK: @__profc__stdin__foo.[[#FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8 -; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [2 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null +; CHECK: @__profd__stdin__foo.[[#FOO_HASH]] = private global { i64, i64, i64, i64, ptr, ptr, i32, [3 x i16], i32 } { i64 {{.*}}, i64 [[#FOO_HASH]], i64 sub (i64 ptrtoint (ptr @__profc__stdin__foo.742261418966908927 to i64), i64 ptrtoint (ptr @__profd__stdin__foo.742261418966908927 to i64)), i64 0, ptr null ; CHECK-NOT: @foo -; CHECK-SAME: , ptr null, i32 1, [2 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 +; CHECK-SAME: , ptr null, i32 1, [3 x i16] zeroinitializer, i32 0 }, section "__llvm_prf_data", comdat($__profc__stdin__foo.[[#FOO_HASH]]), align 8 ; CHECK: @__llvm_prf_nm ; CHECK: @llvm.compiler.used diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw index 9cd225587c92511e99f3497ce1d5f47c6fc5f0af..2327cb76924a7a82aaad3d8d4135b2bbc869a588 100644 GIT binary patch delta 25 hcmaFB|A2o&1mA;`#jD&_{r^ArZ+Ga<(2Z$T>;SUG4XFSC delta 25 hcmaFB|A2o&1fPY~>zYpu|Nr+b_|n8%zcHi3d^BkWXQ;q~{}8~jee0hktN#DrJkOIkI+TF{ zX0YI^%?f`vOg;fr_5L!KFBeQb%shva5cM!VOdpINJ<~YH=c-N(O#cd~eK7d|0{XA2 zYFH&6%DWHJCbaDydjXpM1gQQWo?dWwGr?0yS0{Tm3_5AzQ$ z+Q7KtR(HRVzu%dYp1!6!$!AXbT=MqY*4O{3u}gA_;W2kfsb$ZfsH;9ZvV7_@)#;23 r{WSu+S$HaLo%TI*hM9pynsFJ}wH81UW(Uaqj8G0Nd|-00@P_dLvBrhT literal 0 HcmV?d00001 diff --git a/llvm/test/tools/llvm-profdata/binary-ids-padding.test b/llvm/test/tools/llvm-profdata/binary-ids-padding.test index eda63203a304a..61881b69cfd5c 100644 --- a/llvm/test/tools/llvm-profdata/binary-ids-padding.test +++ b/llvm/test/tools/llvm-profdata/binary-ids-padding.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // There will be 2 20-byte binary IDs, so the total Binary IDs size will be 64 bytes. // 2 * 8 binary ID sizes // + 2 * 20 binary IDs (of size 20) @@ -32,6 +34,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Binary IDs - There are only two in this case that are 20 bytes. RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/large-binary-id-size.test b/llvm/test/tools/llvm-profdata/large-binary-id-size.test index 38b838e0d100a..316a9a4c9df4c 100644 --- a/llvm/test/tools/llvm-profdata/large-binary-id-size.test +++ b/llvm/test/tools/llvm-profdata/large-binary-id-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\40\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Check for a corrupted size being too large past the end of the file. RUN: printf '\7\7\7\7\7\7\7\7' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test index c967e850dbe35..8b686d5c50cb7 100644 --- a/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test +++ b/llvm/test/tools/llvm-profdata/malformed-not-space-for-another-header.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test index 2e747f81a6bfa..089afad420622 100644 --- a/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test +++ b/llvm/test/tools/llvm-profdata/malformed-num-counters-zero.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test index 3c23bc7dd0f7f..e404ba4210cc1 100644 --- a/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test +++ b/llvm/test/tools/llvm-profdata/malformed-ptr-to-counter-array.test @@ -10,10 +10,12 @@ // INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) // INSTR_PROF_RAW_HEADER(uint64_t, BitmaskDelta, (uintptr_t)BitmaskBegin) // INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +// INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) +// INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) // INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -26,6 +28,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Data Section // diff --git a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test index 4a5c42843ff4d..ee54bfb978567 100644 --- a/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test +++ b/llvm/test/tools/llvm-profdata/misaligned-binary-ids-size.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t.profraw // We should fail on this because the binary IDs is not a multiple of 8 bytes. RUN: printf '\77\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw @@ -10,6 +10,8 @@ RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t.profraw // Binary IDs - There are only two in this case that are 20 bytes. RUN: printf '\24\0\0\0\0\0\0\0' >> %t.profraw diff --git a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test index 2a92575ee3407..dfa163f1f3439 100644 --- a/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test +++ b/llvm/test/tools/llvm-profdata/mismatched-raw-profile-header.test @@ -15,6 +15,8 @@ RUN: printf '\0\0\0\0\0\0\0\20' >> %t RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: not llvm-profdata show %t -o /dev/null 2>&1 | FileCheck %s diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test index 8220361df6cfa..63782c8b94d4a 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-be.test @@ -1,5 +1,6 @@ +// Header RUN: printf '\377lprofR\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\12' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +13,8 @@ RUN: printf '\0\0\0\0\1\0\0\0' >> %t RUN: printf '\0\0\0\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t @@ -20,9 +23,8 @@ RUN: printf '\3\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\3' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t @@ -31,9 +33,8 @@ RUN: printf '\2\xff\xff\xd3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\2' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test index 9352ae132380d..e9569bec1178b 100644 --- a/llvm/test/tools/llvm-profdata/raw-32-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-32-bits-le.test @@ -1,5 +1,5 @@ RUN: printf '\201Rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\0\0\0' >> %t RUN: printf '\0\0\0\3\0\0\0\0' >> %t RUN: printf '\0\0\0\2\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t @@ -20,9 +22,8 @@ RUN: printf '\0\0\0\3' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t @@ -31,9 +32,8 @@ RUN: printf '\xd3\xff\xff\2' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0' >> %t RUN: printf '\2\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test index c3e995add6ff2..0bc579eec58ab 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-be.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-be.test @@ -1,5 +1,5 @@ RUN: printf '\377lprofr\201' > %t -RUN: printf '\0\0\0\0\0\0\0\11' >> %t +RUN: printf '\0\0\0\0\0\0\0\12' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\2' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\0\1\0\4\0\0' >> %t RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\2\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\134\370\302\114\333\030\275\254' >> %t RUN: printf '\0\0\0\0\0\0\0\1' >> %t @@ -20,9 +22,8 @@ RUN: printf '\0\0\0\3\0\4\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\3' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\344\023\165\112\031\035\265\067' >> %t RUN: printf '\0\0\0\0\0\0\0\02' >> %t @@ -31,9 +32,8 @@ RUN: printf '\0\0\0\3\0\3\xff\xc3' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\02' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\1' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\023' >> %t RUN: printf '\0\0\0\0\0\0\0\067' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test index 0b3ef2a89abe5..ca9ea54c3f014 100644 --- a/llvm/test/tools/llvm-profdata/raw-64-bits-le.test +++ b/llvm/test/tools/llvm-profdata/raw-64-bits-le.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t -RUN: printf '\11\0\0\0\0\0\0\0' >> %t +RUN: printf '\12\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\2\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\4\0\2\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\254\275\030\333\114\302\370\134' >> %t RUN: printf '\1\0\0\0\0\0\0\0' >> %t @@ -20,9 +22,8 @@ RUN: printf '\0\0\4\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\3\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\067\265\035\031\112\165\023\344' >> %t RUN: printf '\02\0\0\0\0\0\0\0' >> %t @@ -31,9 +32,8 @@ RUN: printf '\xc3\xff\3\0\3\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\02\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t +RUN: printf '\0\0\0\0\0\0\0\0' >> %t RUN: printf '\1\0\0\0' >> %t -RUN: printf '\0\0\0\0' >> %t RUN: printf '\023\0\0\0\0\0\0\0' >> %t RUN: printf '\067\0\0\0\0\0\0\0' >> %t diff --git a/llvm/test/tools/llvm-profdata/raw-two-profiles.test b/llvm/test/tools/llvm-profdata/raw-two-profiles.test index f4a9aa8e1bbc3..70a4210dea9f8 100644 --- a/llvm/test/tools/llvm-profdata/raw-two-profiles.test +++ b/llvm/test/tools/llvm-profdata/raw-two-profiles.test @@ -1,5 +1,5 @@ RUN: printf '\201rforpl\377' > %t-foo.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -12,6 +12,8 @@ RUN: printf '\0\0\4\0\1\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\4\0\2\0\0\0' >> %t-foo.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\254\275\030\333\114\302\370\134' >> %t-foo.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-foo.profraw @@ -26,7 +28,7 @@ RUN: printf '\023\0\0\0\0\0\0\0' >> %t-foo.profraw RUN: printf '\3\0foo\0\0\0' >> %t-foo.profraw RUN: printf '\201rforpl\377' > %t-bar.profraw -RUN: printf '\11\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\12\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\1\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw @@ -39,6 +41,8 @@ RUN: printf '\0\0\6\0\1\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\6\0\2\0\0\0' >> %t-bar.profraw RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw +RUN: printf '\0\0\0\0\0\0\0\0' >> %t-bar.profraw RUN: printf '\067\265\035\031\112\165\023\344' >> %t-bar.profraw RUN: printf '\02\0\0\0\0\0\0\0' >> %t-bar.profraw From a4ec017b1faa15f9f8a49c602eae39793d94112f Mon Sep 17 00:00:00 2001 From: mingmingl Date: Tue, 13 Feb 2024 17:02:15 -0800 Subject: [PATCH 02/13] undo the changes that don't necessarily bundle with profile format change, to get smaller PRs. --- compiler-rt/include/profile/InstrProfData.inc | 7 +- compiler-rt/lib/profile/InstrProfiling.h | 64 ++----- compiler-rt/lib/profile/InstrProfilingMerge.c | 5 + .../lib/profile/InstrProfilingPlatformLinux.c | 9 - llvm/include/llvm/ProfileData/InstrProf.h | 156 +++------------ .../llvm/ProfileData/InstrProfData.inc | 2 + .../llvm/ProfileData/InstrProfReader.h | 13 ++ .../llvm/ProfileData/InstrProfWriter.h | 4 - llvm/lib/ProfileData/InstrProf.cpp | 179 +++++------------- llvm/lib/ProfileData/InstrProfReader.cpp | 37 +++- llvm/lib/ProfileData/InstrProfWriter.cpp | 17 -- .../thinlto_indirect_call_promotion.profraw | Bin 528 -> 544 bytes .../llvm-profdata/Inputs/c-general.profraw | Bin 2016 -> 2032 bytes 13 files changed, 146 insertions(+), 347 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index f0bc2d960ce68..05b34e17f4770 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -208,14 +208,14 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") -/* For virtual table address profiling, the addresses of the virtual table +/* For virtual table address profiling, the address point of the virtual table * (i.e., the address contained in objects pointing to a virtual table) are * profiled. Note this may not be the address of the per C++ class virtual table - * object (i.e., there is an offset). + * object (e.g., there might be an offset). * * The profiled addresses are stored in raw profile, together with the following * two types of information. - * 1. The (beginning and ending) addresses of per C++ class virtual table objects. + * 1. The (starting and ending) addresses of per C++ class virtual table objects. * 2. The (compressed) virtual table object names. * RawInstrProfReader converts profiled virtual table addresses to virtual table * objects' MD5 hash. @@ -983,4 +983,3 @@ InstrProfIsSingleValRange(uint64_t Value) { } #endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ - diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 2ebb501a957bd..9e4526d6dc7e6 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -12,6 +12,17 @@ #include "InstrProfilingPort.h" #include +// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before +// including instr_prof_interface.h so the interface functions are +// declared correctly for the runtime. +// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header, +// because compiler-rt does not support profiling the profiling runtime itself. +#ifndef __LLVM_INSTR_PROFILE_GENERATE +#define __LLVM_INSTR_PROFILE_GENERATE +#endif +#include "profile/instr_prof_interface.h" +#undef __LLVM_INSTR_PROFILE_GENERATE + #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY #include "profile/InstrProfData.inc" @@ -33,7 +44,8 @@ typedef struct __llvm_profile_header { } __llvm_profile_header; typedef struct ValueProfNode * PtrToNodeT; -typedef struct ValueProfNode { +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) + ValueProfNode { #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name; #include "profile/InstrProfData.inc" } ValueProfNode; @@ -110,12 +122,6 @@ VTableProfData *__llvm_profile_begin_vtables(); VTableProfData *__llvm_profile_end_vtables(); uint32_t *__llvm_profile_begin_orderfile(); -/*! - * \brief Clear profile counters to zero. - * - */ -void __llvm_profile_reset_counters(void); - /*! * \brief Merge profile data from buffer. * @@ -166,50 +172,6 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, int __llvm_profile_write_file(void); int __llvm_orderfile_write_file(void); -/*! - * \brief this is a wrapper interface to \c __llvm_profile_write_file. - * After this interface is invoked, an already dumped flag will be set - * so that profile won't be dumped again during program exit. - * Invocation of interface __llvm_profile_reset_counters will clear - * the flag. This interface is designed to be used to collect profile - * data from user selected hot regions. The use model is - * __llvm_profile_reset_counters(); - * ... hot region 1 - * __llvm_profile_dump(); - * .. some other code - * __llvm_profile_reset_counters(); - * ... hot region 2 - * __llvm_profile_dump(); - * - * It is expected that on-line profile merging is on with \c %m specifier - * used in profile filename . If merging is not turned on, user is expected - * to invoke __llvm_profile_set_filename to specify different profile names - * for different regions before dumping to avoid profile write clobbering. - */ -int __llvm_profile_dump(void); - -int __llvm_orderfile_dump(void); - -/*! - * \brief Set the filename for writing instrumentation data. - * - * Sets the filename to be used for subsequent calls to - * \a __llvm_profile_write_file(). - * - * \c Name is not copied, so it must remain valid. Passing NULL resets the - * filename logic to the default behaviour. - * - * Note: There may be multiple copies of the profile runtime (one for each - * instrumented image/DSO). This API only modifies the filename within the - * copy of the runtime available to the calling image. - * - * Warning: This is a no-op if continuous mode (\ref - * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is - * that in continuous mode, profile counters are mmap()'d to the profile at - * program initialization time. Support for transferring the mmap'd profile - * counts to a new file has not been implemented. - */ -void __llvm_profile_set_filename(const char *Name); /*! * \brief Set the FILE object for writing instrumentation data. Return 0 if set diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 53df1279ef961..ad7a50dc77f44 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -41,6 +41,9 @@ uint64_t lprofGetLoadModuleSignature(void) { #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" #endif /* Returns 1 if profile is not structurally compatible. */ @@ -255,4 +258,6 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, #ifdef __GNUC__ #pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop #endif diff --git a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index dabe0e900465e..d2554a2702aaf 100644 --- a/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -20,15 +20,6 @@ #include "InstrProfiling.h" #include "InstrProfilingInternal.h" -#if defined(__FreeBSD__) && !defined(ElfW) -/* - * FreeBSD's elf.h and link.h headers do not define the ElfW(type) macro yet. - * If this is added to all supported FreeBSD versions in the future, this - * compatibility macro can be removed. - */ -#define ElfW(type) __ElfN(type) -#endif - #define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 16073521a976d..1ba577059cc40 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -89,9 +89,6 @@ inline StringRef getInstrProfValueProfMemOpFuncName() { /// Return the name prefix of variables containing instrumented function names. inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } -/// Return the name prefix of variables containing virtual table profile data. -inline StringRef getInstrProfVTableVarPrefix() { return "__profvt_"; } - /// Return the name prefix of variables containing per-function control data. inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } @@ -113,8 +110,6 @@ inline StringRef getInstrProfNamesVarName() { return "__llvm_prf_nm"; } -inline StringRef getInstrProfVTableNamesVarName() { return "__llvm_prf_vnm"; } - /// Return the name of a covarage mapping variable (internal linkage) /// for each instrumented source module. Such variables are allocated /// in the __llvm_covmap section. @@ -200,7 +195,7 @@ std::string getIRPGOFuncName(const Function &F, bool InLTO = false); /// \return the filename and the function name parsed from the output of /// \c getIRPGOFuncName() -std::pair getParsedIRPGOName(StringRef IRPGOFuncName); +std::pair getParsedIRPGOName(StringRef IRPGOName); /// Return the name of the global variable used to store a function /// name in PGO instrumentation. \c FuncName is the IRPGO function name @@ -251,9 +246,6 @@ Error collectGlobalObjectNameStrings(ArrayRef NameStrs, Error collectPGOFuncNameStrings(ArrayRef NameVars, std::string &Result, bool doCompression = true); -Error collectVTableStrings(ArrayRef VTables, - std::string &Result, bool doCompression); - /// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being /// set in IR PGO compilation. bool isIRPGOFlagSet(const Module *M); @@ -296,8 +288,6 @@ inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } /// Return the PGOFuncName meta data associated with a function. MDNode *getPGOFuncNameMetadata(const Function &F); -std::string getPGOName(const GlobalVariable &V, bool InLTO = false); - /// Create the PGOFuncName meta data if PGOFuncName is different from /// function's raw name. This should only apply to internal linkage functions /// declared by users only. @@ -305,7 +295,7 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); /// Check if we can use Comdat for profile variables. This will eliminate /// the duplicated profile variables for Comdat functions. -bool needsComdatForCounter(const GlobalValue &GV, const Module &M); +bool needsComdatForCounter(const Function &F, const Module &M); /// An enum describing the attributes of an instrumented profile. enum class InstrProfKind { @@ -439,42 +429,37 @@ uint64_t ComputeHash(StringRef K); class InstrProfSymtab { public: using AddrHashMap = std::vector>; - using RangeHashMap = - std::vector, uint64_t>>; private: StringRef Data; uint64_t Address = 0; - // Unique name strings. Used to ensure entries in MD5NameMap (a vector that's - // going to be sorted) has unique MD5 keys in the first place. + // Unique name strings. StringSet<> NameTab; - // Records the unique virtual table names. This is used by InstrProfWriter to - // write out an on-disk chained hash table of virtual table names. - // InstrProfWriter stores per function profile data (keyed by function names) - // so it doesn't use a StringSet for function names. - StringSet<> VTableNames; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; - // A map from MD5 keys to virtual table definitions. Only populated when - // building the Symtab from a module. - std::vector> MD5VTableMap; // A map from MD5 keys to function define. We only populate this map // when build the Symtab from a Module. std::vector> MD5FuncMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; - // A map from virtual table runtime address to function name MD5 hash. - // This map is only populated and used by raw instr profile reader. - // This is a different map from 'AddrToMD5Map' for readability and - // debuggability. - RangeHashMap VTableAddrRangeToMD5Map; bool Sorted = false; static StringRef getExternalSymbol() { return "** External Symbol **"; } + // Returns the canonial name of the given PGOName. In a canonical name, all + // suffixes that begins with "." except ".__uniq." are stripped. + // FIXME: Unify this with `FunctionSamples::getCanonicalFnName`. + static StringRef getCanonicalName(StringRef PGOName); + + // Add the function into the symbol table, by creating the following + // map entries: + // name-set = {PGOFuncName} + {getCanonicalName(PGOFuncName)} if the canonical + // name is different from pgo name + // - In MD5NameMap: for name in name-set + // - In MD5FuncMap: for name in name-set Error addFuncWithName(Function &F, StringRef PGOFuncName); // If the symtab is created by a series of calls to \c addFuncName, \c @@ -496,19 +481,9 @@ class InstrProfSymtab { /// \c NameStrings is a string composed of one of more sub-strings /// encoded in the format described in \c collectPGOFuncNameStrings. - /// This method is a wrapper to \c readAndDecodeStrings method. + /// This method is a wrapper to \c readPGOFuncNameStrings method. Error create(StringRef NameStrings); - /// \c FuncNameStrings is a string composed of one or more encoded function - /// name strings, and \c VTableNameStrings composes of one or more encoded - /// vtable names. This function is a wrapper to \c readAndDecodeStrings - /// method. - Error create(StringRef FuncNameStrings, StringRef VTableNameStrings); - - /// Initialize 'this' with the set of vtable names encoded in - /// \c CompressedVTableNames. - Error initVTableNamesFromCompressedStrings(StringRef CompressedVTableNames); - /// This interface is used by reader of CoverageMapping test /// format. inline Error create(StringRef D, uint64_t BaseAddr); @@ -521,70 +496,32 @@ class InstrProfSymtab { /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. - template - Error create(const NameIterRange &IterRange); - - /// Create InstrProfSymtab from a set of function names and vtable - /// names iteratable from \p IterRange. This interface is used by - /// IndexedProfReader. - template - Error create(const FuncNameIterRange &FuncIterRange, - const VTableNameIterRange &VTableIterRange); - - Error addSymbolName(StringRef SymbolName) { - if (SymbolName.empty()) - return make_error(instrprof_error::malformed, - "symbol name is empty"); + template Error create(const NameIterRange &IterRange); - // Insert into NameTab so that MD5NameMap (a vector that will be sorted) - // won't have duplicated entries in the first place. - auto Ins = NameTab.insert(SymbolName); + /// Update the symtab by adding \p FuncName to the table. This interface + /// is used by the raw and text profile readers. + Error addFuncName(StringRef FuncName) { + if (FuncName.empty()) + return make_error(instrprof_error::malformed, + "function name is empty"); + auto Ins = NameTab.insert(FuncName); if (Ins.second) { MD5NameMap.push_back(std::make_pair( - IndexedInstrProf::ComputeHash(SymbolName), Ins.first->getKey())); + IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); Sorted = false; } return Error::success(); } - /// The method name is kept since there are many callers. - /// It just forwards to 'addSymbolName'. - Error addFuncName(StringRef FuncName) { return addSymbolName(FuncName); } - - /// Adds VTableName as a known symbol, and inserts it to a map that - /// tracks all vtable names. - Error addVTableName(StringRef VTableName) { - if (Error E = addSymbolName(VTableName)) - return E; - - // Record VTableName. InstrProfWriter uses this map. The comment around - // class member explains why. - VTableNames.insert(VTableName); - return Error::success(); - } - - const StringSet<> &getVTableNames() const { return VTableNames; } - /// Map a function address to its name's MD5 hash. This interface /// is only used by the raw profiler reader. void mapAddress(uint64_t Addr, uint64_t MD5Val) { AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); } - /// Map the address range (i.e., [start_address, end_address]) of a variable - /// to its names' MD5 hash. This interface is only used by the raw profile - /// reader. - void mapVTableAddress(uint64_t StartAddr, uint64_t EndAddr, uint64_t MD5Val) { - VTableAddrRangeToMD5Map.push_back( - std::make_pair(std::make_pair(StartAddr, EndAddr), MD5Val)); - } - /// Return a function's hash, or 0, if the function isn't in this SymTab. uint64_t getFunctionHashFromAddress(uint64_t Address); - /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab. - uint64_t getVTableHashFromAddress(uint64_t Address); - /// Return function's PGO name from the function name's symbol /// address in the object file. If an error occurs, return /// an empty string. @@ -606,8 +543,6 @@ class InstrProfSymtab { /// Return function from the name's md5 hash. Return nullptr if not found. inline Function *getFunction(uint64_t FuncMD5Hash); - // Return vtable from the name's MD5 hash. Return nullptr if not found. - inline GlobalVariable *getGlobalVariable(uint64_t GlobalVariableMD5Hash); /// Return the name section data. inline StringRef getNameData() const { return Data; } @@ -632,23 +567,6 @@ Error InstrProfSymtab::create(const NameIterRange &IterRange) { return Error::success(); } -template -Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange, - const VTableNameIterRange &VTableIterRange) { - for (auto Name : FuncIterRange) - if (Error E = addFuncName(Name)) - return E; - - for (auto VTableName : VTableIterRange) { - if (Error E = addVTableName(VTableName)) { - return E; - } - } - - finalizeSymtab(); - return Error::success(); -} - void InstrProfSymtab::finalizeSymtab() { if (Sorted) return; @@ -657,13 +575,6 @@ void InstrProfSymtab::finalizeSymtab() { llvm::sort(AddrToMD5Map, less_first()); AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), AddrToMD5Map.end()); - // VTable object address ranges should not overlap; so sort by either - // beginning address or end address is fine. - llvm::sort(VTableAddrRangeToMD5Map, less_first()); - // std::unique uses == operator for std::pair. - VTableAddrRangeToMD5Map.erase(std::unique(VTableAddrRangeToMD5Map.begin(), - VTableAddrRangeToMD5Map.end()), - VTableAddrRangeToMD5Map.end()); Sorted = true; } @@ -694,19 +605,6 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { return nullptr; } -GlobalVariable * -InstrProfSymtab::getGlobalVariable(uint64_t GlobalVariableMD5Hash) { - finalizeSymtab(); - auto Result = - llvm::lower_bound(MD5VTableMap, GlobalVariableMD5Hash, - [](const std::pair &LHS, - uint64_t RHS) { return LHS.first < RHS; }); - - if (Result != MD5VTableMap.end() && Result->first == GlobalVariableMD5Hash) - return Result->second; - return nullptr; -} - // To store the sums of profile count values, or the percentage of // the sums of the total count values. struct CountSumOrPercent { @@ -972,8 +870,6 @@ struct InstrProfRecord { return ValueData->IndirectCallSites; case IPVK_MemOPSize: return ValueData->MemOPSizes; - case IPVK_VTableTarget: - return ValueData->VTableTargets; default: llvm_unreachable("Unknown value kind!"); } @@ -1155,7 +1051,8 @@ const HashT HashType = HashT::MD5; inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } // This structure defines the file header of the LLVM profile -// data file in indexed-format. +// data file in indexed-format. Please update llvm/docs/InstrProfileFormat.rst +// as appropriate when updating the indexed profile format. struct Header { uint64_t Magic; uint64_t Version; @@ -1310,7 +1207,6 @@ template struct alignas(8) VTableProfileData { #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name; #include "llvm/ProfileData/InstrProfData.inc" }; - // File header structure of the LLVM profile data in raw format. // The definition should match the header referenced in // compiler-rt/lib/profile/InstrProfilingFile.c and diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index f6410d6696f49..77720aba3eb48 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -139,6 +139,8 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ /* INSTR_PROF_RAW_HEADER start */ /* Definition of member fields of the raw profile header data structure. */ +/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating + raw profile format. */ #ifndef INSTR_PROF_RAW_HEADER #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) #else diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 87f15639a2c3c..636f11342887b 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -326,12 +326,16 @@ class RawInstrProfReader : public InstrProfReader { uint64_t NamesDelta; const RawInstrProf::ProfileData *Data; const RawInstrProf::ProfileData *DataEnd; + const RawInstrProf::VTableProfileData *VTableBegin = nullptr; + const RawInstrProf::VTableProfileData *VTableEnd = nullptr; const char *CountersStart; const char *CountersEnd; const char *BitmapStart; const char *BitmapEnd; const char *NamesStart; const char *NamesEnd; + const char *VNamesStart = nullptr; + const char *VNamesEnd = nullptr; // After value profile is all read, this pointer points to // the header of next profile data (if exists) const uint8_t *ValueDataStart; @@ -656,6 +660,15 @@ class IndexedInstrProfReader : public InstrProfReader { std::unique_ptr MemProfRecordTable; /// MemProf frame profile data on-disk indexed via frame id. std::unique_ptr MemProfFrameTable; + /// VTableNamePtr points to the beginning of compressed vtable names. + /// When a symtab is constructed from profiles by llvm-profdata, the list of + /// names could be decompressed based on `VTableNamePtr` and + /// `CompressedVTableNamesLen`. + /// A compiler that reads indexed profiles could construct symtab from module + /// IR so it doesn't need the decompressed names. + const char *VTableNamePtr = nullptr; + /// The length of compressed vtable names. + uint64_t CompressedVTableNamesLen = 0; /// Total size of binary ids. uint64_t BinaryIdsSize{0}; /// Start address of binary id length and data pairs. diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h index 049fa36bb53f5..047b14f223bd9 100644 --- a/llvm/include/llvm/ProfileData/InstrProfWriter.h +++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h @@ -63,9 +63,6 @@ class InstrProfWriter { // List of binary ids. std::vector BinaryIds; - // Read the vtable names from raw instr profile reader. - StringSet<> VTableNames; - // An enum describing the attributes of the profile. InstrProfKind ProfileKind = InstrProfKind::Unknown; // Use raw pointer here for the incomplete type object. @@ -87,7 +84,6 @@ class InstrProfWriter { void addRecord(NamedInstrProfRecord &&I, function_ref Warn) { addRecord(std::move(I), 1, Warn); } - void addVTableName(StringRef VTableName) { VTableNames.insert(VTableName); } /// Add \p SrcTraces using reservoir sampling where \p SrcStreamSize is the /// total number of temporal profiling traces the source has seen. diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp index 96a1df04edd47..b9afee413853e 100644 --- a/llvm/lib/ProfileData/InstrProf.cpp +++ b/llvm/lib/ProfileData/InstrProf.cpp @@ -14,7 +14,6 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SetVector.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringRef.h" @@ -27,7 +26,6 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Mangler.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" @@ -221,12 +219,6 @@ cl::opt DoInstrProfNameCompression( "enable-name-compression", cl::desc("Enable name/filename string compression"), cl::init(true)); -cl::opt EnableVTableValueProfiling( - "enable-vtable-value-profiling", cl::init(false), - cl::desc("If true, the virtual table address will be instrumented to know " - "the types of a C++ pointer. The information is used in indirect " - "call promotion to do selective vtable-based comparison.")); - std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo) { @@ -303,29 +295,20 @@ static StringRef getStrippedSourceFileName(const GlobalObject &GO) { return FileName; } -// The PGO name has the format [;] where ; is -// provided if linkage is local and is the mangled function -// name. The filepath is used to discriminate possibly identical function names. -// ; is used because it is unlikely to be found in either or -// . +// The PGO name has the format [;] where ; is +// provided if linkage is local and is used to discriminate possibly identical +// mangled names. ";" is used because it is unlikely to be found in either +// or . // // Older compilers used getPGOFuncName() which has the format -// [:]. is used to discriminate between -// possibly identical function names when linkage is local and -// simply comes from F.getName(). This caused trouble for Objective-C functions -// which commonly have :'s in their names. Also, since is not -// mangled, they cannot be passed to Mach-O linkers via -order_file. We still -// need to compute this name to lookup functions from profiles built by older -// compilers. +// [:]. This caused trouble for Objective-C functions +// which commonly have :'s in their names. We still need to compute this name to +// lookup functions from profiles built by older compilers. static std::string getIRPGONameForGlobalObject(const GlobalObject &GO, GlobalValue::LinkageTypes Linkage, StringRef FileName) { - SmallString<64> Name; - // FIXME: Mangler's handling is kept outside of `getGlobalIdentifier` for now. - // For more details please check issue #74565. - Mangler().getNameWithPrefix(Name, &GO, /*CannotUsePrivateLabel=*/true); - return GlobalValue::getGlobalIdentifier(Name, Linkage, FileName); + return GlobalValue::getGlobalIdentifier(GO.getName(), Linkage, FileName); } static std::optional lookupPGONameFromMetadata(MDNode *MD) { @@ -395,19 +378,12 @@ std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { return getPGOFuncName(F.getName(), GlobalValue::ExternalLinkage, ""); } -std::string getPGOName(const GlobalVariable &V, bool InLTO) { - // PGONameMetadata should be set by compiler at profile use time - // and read by symtab creation to look up symbols corresponding to - // a MD5 hash. - return getIRPGOObjectName(V, InLTO, nullptr /* PGONameMetadata */); -} - -// See getIRPGOFuncName() for a discription of the format. -std::pair getParsedIRPGOName(StringRef IRPGOFuncName) { - auto [FileName, FuncName] = IRPGOFuncName.split(';'); - if (FuncName.empty()) - return std::make_pair(StringRef(), IRPGOFuncName); - return std::make_pair(FileName, FuncName); +// See getIRPGOObjectName() for a discription of the format. +std::pair getParsedIRPGOName(StringRef IRPGOName) { + auto [FileName, MangledName] = IRPGOName.split(kGlobalIdentifierDelimiter); + if (MangledName.empty()) + return std::make_pair(StringRef(), IRPGOName); + return std::make_pair(FileName, MangledName); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -483,17 +459,6 @@ Error InstrProfSymtab::create(Module &M, bool InLTO) { if (Error E = addFuncWithName(F, getPGOFuncName(F, InLTO))) return E; } - - SmallVector Types; - for (GlobalVariable &G : M.globals()) { - if (!G.hasName()) - continue; - Types.clear(); - G.getMetadata(LLVMContext::MD_type, Types); - if (!Types.empty()) { - MD5VTableMap.emplace_back(G.getGUID(), &G); - } - } Sorted = false; finalizeSymtab(); return Error::success(); @@ -552,77 +517,47 @@ Error InstrProfSymtab::create(StringRef NameStrings) { std::bind(&InstrProfSymtab::addFuncName, this, std::placeholders::_1)); } -Error InstrProfSymtab::create(StringRef FuncNameStrings, - StringRef VTableNameStrings) { - if (Error E = readAndDecodeStrings(FuncNameStrings, - std::bind(&InstrProfSymtab::addFuncName, - this, std::placeholders::_1))) - return E; - - return readAndDecodeStrings( - VTableNameStrings, - std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); -} - -Error InstrProfSymtab::initVTableNamesFromCompressedStrings( - StringRef CompressedVTableStrings) { - return readAndDecodeStrings( - CompressedVTableStrings, - std::bind(&InstrProfSymtab::addVTableName, this, std::placeholders::_1)); -} - -Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { - if (Error E = addFuncName(PGOFuncName)) - return E; - MD5FuncMap.emplace_back(Function::getGUID(PGOFuncName), &F); +StringRef InstrProfSymtab::getCanonicalName(StringRef PGOName) { // In ThinLTO, local function may have been promoted to global and have // suffix ".llvm." added to the function name. We need to add the // stripped function name to the symbol table so that we can find a match // from profile. // - // We may have other suffixes similar as ".llvm." which are needed to - // be stripped before the matching, but ".__uniq." suffix which is used - // to differentiate internal linkage functions in different modules - // should be kept. Now this is the only suffix with the pattern ".xxx" - // which is kept before matching. + // ".__uniq." suffix is used to differentiate internal linkage functions in + // different modules and should be kept. This is the only suffix with the + // pattern ".xxx" which is kept before matching, other suffixes similar as + // ".llvm." will be stripped. const std::string UniqSuffix = ".__uniq."; - auto pos = PGOFuncName.find(UniqSuffix); - // Search '.' after ".__uniq." if ".__uniq." exists, otherwise - // search '.' from the beginning. - if (pos != std::string::npos) + size_t pos = PGOName.find(UniqSuffix); + if (pos != StringRef::npos) pos += UniqSuffix.length(); else pos = 0; - pos = PGOFuncName.find('.', pos); - if (pos != std::string::npos && pos != 0) { - StringRef OtherFuncName = PGOFuncName.substr(0, pos); - if (Error E = addFuncName(OtherFuncName)) - return E; - MD5FuncMap.emplace_back(Function::getGUID(OtherFuncName), &F); - } - return Error::success(); + + // Search '.' after ".__uniq." if ".__uniq." exists, otherwise search '.' from + // the beginning. + pos = PGOName.find('.', pos); + if (pos != StringRef::npos && pos != 0) + return PGOName.substr(0, pos); + + return PGOName; } -uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) { - finalizeSymtab(); - auto It = lower_bound( - VTableAddrRangeToMD5Map, Address, - [](std::pair, uint64_t> VTableRangeAddr, - uint64_t Addr) { - // Find the first address range of which end address is larger than - // `Addr`. Smaller-than-or-equal-to is used because the profiled address - // within a vtable should be [start-address, end-address). - return VTableRangeAddr.first.second <= Addr; - }); - - // Returns the MD5 hash if Address is within the address range of an entry. - if (It != VTableAddrRangeToMD5Map.end() && It->first.first <= Address) { - return It->second; - } - // The virtual table address collected from value profiler could be defined - // in another module that is not instrumented. Force the value to be 0 in - // this case. - return 0; +Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName) { + auto mapName = [&](StringRef Name) -> Error { + if (Error E = addFuncName(Name)) + return E; + MD5FuncMap.emplace_back(Function::getGUID(Name), &F); + return Error::success(); + }; + if (Error E = mapName(PGOFuncName)) + return E; + + StringRef CanonicalFuncName = getCanonicalName(PGOFuncName); + if (CanonicalFuncName != PGOFuncName) + return mapName(CanonicalFuncName); + + return Error::success(); } uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) { @@ -701,17 +636,6 @@ Error collectPGOFuncNameStrings(ArrayRef NameVars, NameStrs, compression::zlib::isAvailable() && doCompression, Result); } -Error collectVTableStrings(ArrayRef VTables, - std::string &Result, bool doCompression) { - std::vector VTableNameStrs; - for (auto *VTable : VTables) { - VTableNameStrs.push_back(getPGOName(*VTable)); - } - return collectGlobalObjectNameStrings( - VTableNameStrs, compression::zlib::isAvailable() && doCompression, - Result); -} - void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const { uint64_t FuncSum = 0; Sum.NumEntries += Counts.size(); @@ -974,9 +898,6 @@ uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind, if (ValueKind == IPVK_IndirectCallTarget) return SymTab->getFunctionHashFromAddress(Value); - if (ValueKind == IPVK_VTableTarget) - return SymTab->getVTableHashFromAddress(Value); - return Value; } @@ -1367,8 +1288,8 @@ void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName) { F.setMetadata(getPGOFuncNameMetadataName(), N); } -bool needsComdatForCounter(const GlobalValue &GV, const Module &M) { - if (GV.hasComdat()) +bool needsComdatForCounter(const Function &F, const Module &M) { + if (F.hasComdat()) return true; if (!Triple(M.getTargetTriple()).supportsCOMDAT()) @@ -1384,7 +1305,7 @@ bool needsComdatForCounter(const GlobalValue &GV, const Module &M) { // available_externally functions will end up being duplicated in raw profile // data. This can result in distorted profile as the counts of those dups // will be accumulated by the profile merger. - GlobalValue::LinkageTypes Linkage = GV.getLinkage(); + GlobalValue::LinkageTypes Linkage = F.getLinkage(); if (Linkage != GlobalValue::ExternalWeakLinkage && Linkage != GlobalValue::AvailableExternallyLinkage) return false; @@ -1540,7 +1461,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) { if (Base.ValueCounts[I] < 1.0f && Test.ValueCounts[I] < 1.0f) continue; - char ProfileKindName[20] = {0}; + char ProfileKindName[20]; switch (I) { case IPVK_IndirectCallTarget: strncpy(ProfileKindName, "IndirectCall", 19); @@ -1548,9 +1469,6 @@ void OverlapStats::dump(raw_fd_ostream &OS) const { case IPVK_MemOPSize: strncpy(ProfileKindName, "MemOP", 19); break; - case IPVK_VTableTarget: - strncpy(ProfileKindName, "VTable", 19); - break; default: snprintf(ProfileKindName, 19, "VP[%d]", I); break; @@ -1653,6 +1571,7 @@ size_t Header::size() const { case 12ull: return offsetOf(&Header::VTableNamesOffset) + sizeof(Header::VTableNamesOffset); + [[fallthrough]]; case 11ull: [[fallthrough]]; case 10ull: diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 0d8d43daae960..98d5db31b7e7f 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -366,6 +366,8 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { return E; Value = IndexedInstrProf::ComputeHash(VD.first); } + } else if (ValueKind == IPVK_VTableTarget) { + // do nothing } else { READ_NUM(VD.first, Value); } @@ -582,10 +584,17 @@ Error RawInstrProfReader::readHeader( auto NumBitmapBytes = swap(Header.NumBitmapBytes); auto PaddingBytesAfterBitmapBytes = swap(Header.PaddingBytesAfterBitmapBytes); auto NamesSize = swap(Header.NamesSize); + auto VTableNameSize = swap(Header.VNamesSize); + auto NumVTables = swap(Header.NumVTables); ValueKindLast = swap(Header.ValueKindLast); auto DataSize = NumData * sizeof(RawInstrProf::ProfileData); - auto PaddingSize = getNumPaddingBytes(NamesSize); + auto PaddingBytesAfterNames = getNumPaddingBytes(NamesSize); + auto PaddingBytesAfterVTableNames = getNumPaddingBytes(VTableNameSize); + + auto VTableSectionSize = + NumVTables * sizeof(RawInstrProf::VTableProfileData); + auto PaddingBytesAfterVTableProfData = getNumPaddingBytes(VTableSectionSize); // Profile data starts after profile header and binary ids if exist. ptrdiff_t DataOffset = sizeof(RawInstrProf::Header) + BinaryIdSize; @@ -594,7 +603,12 @@ Error RawInstrProfReader::readHeader( CountersOffset + CountersSize + PaddingBytesAfterCounters; ptrdiff_t NamesOffset = BitmapOffset + NumBitmapBytes + PaddingBytesAfterBitmapBytes; - ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; + ptrdiff_t VTableProfDataOffset = + NamesOffset + NamesSize + PaddingBytesAfterNames; + ptrdiff_t VTableNameOffset = VTableProfDataOffset + VTableSectionSize + + PaddingBytesAfterVTableProfData; + ptrdiff_t ValueDataOffset = + VTableNameOffset + VTableNameSize + PaddingBytesAfterVTableNames; auto *Start = reinterpret_cast(&Header); if (Start + ValueDataOffset > DataBuffer->getBufferEnd()) @@ -614,8 +628,14 @@ Error RawInstrProfReader::readHeader( Data = reinterpret_cast *>( Start + DataOffset); DataEnd = Data + NumData; + VTableBegin = + reinterpret_cast *>( + Start + VTableProfDataOffset); + VTableEnd = VTableBegin + NumVTables; NamesStart = Start + NamesOffset; NamesEnd = NamesStart + NamesSize; + VNamesStart = Start + VTableNameOffset; + VNamesEnd = VNamesStart + VTableNameSize; } CountersStart = Start + CountersOffset; @@ -1260,6 +1280,19 @@ Error IndexedInstrProfReader::readHeader() { "corrupted binary ids"); } + if (GET_VERSION(Header->formatVersion()) >= 12) { + uint64_t VTableNamesOffset = + endian::byte_swap( + Header->VTableNamesOffset); + const unsigned char *Ptr = Start + VTableNamesOffset; + + CompressedVTableNamesLen = + support::endian::readNext(Ptr); + + VTableNamePtr = (const char *)Ptr; + } + if (GET_VERSION(Header->formatVersion()) >= 10 && Header->formatVersion() & VARIANT_MASK_TEMPORAL_PROF) { uint64_t TemporalProfTracesOffset = diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 7592c0ffd3272..79112d6a4631d 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -616,18 +616,6 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { std::string CompressedVTableNames; - std::vector VTableNameStrs; - for (const auto &VTableName : VTableNames.keys()) { - VTableNameStrs.push_back(VTableName.str()); - } - - if (!VTableNameStrs.empty()) { - if (Error E = collectGlobalObjectNameStrings( - VTableNameStrs, compression::zlib::isAvailable(), - CompressedVTableNames)) - return E; - } - uint64_t CompressedStringLen = CompressedVTableNames.length(); // Record the length of compressed string. @@ -830,11 +818,6 @@ Error InstrProfWriter::writeText(raw_fd_ostream &OS) { } } - for (const auto &VTableName : VTableNames) { - if (Error E = Symtab.addVTableName(VTableName.getKey())) - return E; - } - if (static_cast(ProfileKind & InstrProfKind::TemporalProfile)) writeTextTemporalProfTraceData(OS, Symtab); diff --git a/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw b/llvm/test/Transforms/PGOProfile/Inputs/thinlto_indirect_call_promotion.profraw index 5efda10bb98a941c04b6846db05d3691bc36aac0..3daa98f937b691880ffff203c9426bfacddf749d 100644 GIT binary patch delta 133 zcmbQhvVeuNu_!ISs37M**F;W##f(djpGdFz|9^9xwDglu1`NP7F;ks2U=>hu;#6za s1Tf>OHE#ik0aQLiPe%I5WLZXI)&n4s$)Sw16~Kysa*R;Jz`Bw604`-Eq5uE@ delta 117 zcmZ3$GJ%D&u_!ISs37M*=R{6_L67IVA1SZ;|9^9yv+SKv1_s87mFlblGl86mORZTI rz>KHXyapf!P9f diff --git a/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw b/llvm/test/tools/llvm-profdata/Inputs/c-general.profraw index 2327cb76924a7a82aaad3d8d4135b2bbc869a588..a3e884343942ebc70ba95ab4ee006630b6816d80 100644 GIT binary patch delta 39 vcmaFB|AC*gu_!ISs37M**F??$Ud Date: Tue, 13 Feb 2024 23:33:44 -0800 Subject: [PATCH 03/13] fix the alignment of struct --- compiler-rt/lib/profile/InstrProfiling.h | 5 ++--- llvm/include/llvm/ProfileData/InstrProfData.inc | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index 9e4526d6dc7e6..be694a8d3330b 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -44,14 +44,13 @@ typedef struct __llvm_profile_header { } __llvm_profile_header; typedef struct ValueProfNode * PtrToNodeT; -typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) - ValueProfNode { +typedef struct ValueProfNode { #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name; #include "profile/InstrProfData.inc" } ValueProfNode; typedef void *IntPtrT; -typedef struct VTableProfData { +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData { #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name; #include "profile/InstrProfData.inc" } VTableProfData; diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 77720aba3eb48..8c50006e85038 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -370,7 +370,7 @@ typedef struct ValueProfRecord { /*! * Return the number of value sites. */ - uint32_t getNumValueSites() const { return NumValueSites; } + uint32_t getNumValueSites() const { return NumValueSites; } /*! * Read data from this record and save it to Record. */ From d56bb34434bc188a0a5d1e8379453f5c7b920ed9 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 14 Feb 2024 14:56:43 -0800 Subject: [PATCH 04/13] polish format --- compiler-rt/include/profile/InstrProfData.inc | 14 +++++----- .../lib/profile/InstrProfilingBuffer.c | 26 ++++++++++++++----- .../lib/profile/InstrProfilingInternal.h | 4 ++- compiler-rt/lib/profile/InstrProfilingMerge.c | 3 +-- .../profile/instrprof-write-buffer-internal.c | 6 +++-- llvm/include/llvm/ProfileData/InstrProf.h | 3 ++- .../llvm/ProfileData/InstrProfReader.h | 2 +- llvm/lib/ProfileData/InstrProfReader.cpp | 5 +++- llvm/lib/ProfileData/InstrProfWriter.cpp | 7 +++-- 9 files changed, 47 insertions(+), 23 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 05b34e17f4770..fcaa14d6419d0 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -103,14 +103,13 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #else #define INSTR_PROF_VTABLE_DATA_DEFINED #endif -INSTR_PROF_VTABLE_DATA( - const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, - ConstantInt::get(llvm::Type::getInt64Ty(Ctx), +INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ + VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ IndexedInstrProf::ComputeHash(PGOVTableName))) -INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ VTablePointer, VTableAddr) -INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ VTableSizeVal)) #undef INSTR_PROF_VTABLE_DATA /* INSTR_PROF_VTABLE_DATA end. */ @@ -306,7 +305,6 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED - INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ INSTR_PROF_DATA_COFF, "__DATA,") @@ -350,6 +348,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covname, \ #undef INSTR_PROF_SECT_ENTRY #endif + #ifdef INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_DATA_DEFINED @@ -521,6 +520,7 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #undef INSTR_PROF_VALUE_PROF_DATA #endif /* INSTR_PROF_VALUE_PROF_DATA */ + #ifdef INSTR_PROF_COMMON_API_IMPL #define INSTR_PROF_DATA_DEFINED #ifdef __cplusplus diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index f31dc7d4e2111..a38d0bb96f11e 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -51,10 +51,14 @@ uint64_t __llvm_profile_get_size_for_buffer(void) { const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return __llvm_profile_get_size_for_buffer_internal( DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, - NamesBegin, NamesEnd); + NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd); } COMPILER_RT_VISIBILITY @@ -70,6 +74,7 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } + COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, const VTableProfData *End) { @@ -179,7 +184,9 @@ COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) { + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); @@ -187,22 +194,29 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( __llvm_profile_get_counters_size(CountersBegin, CountersEnd); const uint64_t NumBitmapBytes = __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t VTableSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNameSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; __llvm_profile_get_padding_sizes_for_counters( DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, 0 /* VNameSize */, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, - &PaddingBytesAfterNames, NULL /* PaddingBytesAfterVTable */, - NULL /* PaddingbytesAfterVNames */); + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames); return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize + PaddingBytesBeforeCounters + CountersSize + PaddingBytesAfterCounters + NumBitmapBytes + - PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames; + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames + + VTableSize + PaddingBytesAfterVTable + VNameSize + + PaddingBytesAfterVNames; } COMPILER_RT_VISIBILITY diff --git a/compiler-rt/lib/profile/InstrProfilingInternal.h b/compiler-rt/lib/profile/InstrProfilingInternal.h index 38159b668a1df..d5bd0e41fb129 100644 --- a/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -22,7 +22,9 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, - const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd); + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index ad7a50dc77f44..33ffd9636dcb7 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -113,8 +113,7 @@ getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { // for runtime profile merge. To merge runtime addresses from multiple // profiles collected from the same instrumented binary, the binary should be // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR - // disabled). - // In this set-up these three sections remain unchanged. + // disabled). In this set-up these three sections remain unchanged. const uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData); const uint64_t PaddingBytesAfterVTableSection = diff --git a/compiler-rt/test/profile/instrprof-write-buffer-internal.c b/compiler-rt/test/profile/instrprof-write-buffer-internal.c index d9670f739ca98..2c1c29ac0c588 100644 --- a/compiler-rt/test/profile/instrprof-write-buffer-internal.c +++ b/compiler-rt/test/profile/instrprof-write-buffer-internal.c @@ -31,7 +31,8 @@ char *__llvm_profile_end_bitmap(void); uint64_t __llvm_profile_get_size_for_buffer_internal( const void *DataBegin, const void *DataEnd, const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, - const char *NamesBegin, const char *NamesEnd); + const char *NamesBegin, const char *NamesEnd, const void *VTableBegin, + const void *VTableEnd, const char *VNamesBegin, const char *VNamesEnd); int __llvm_profile_write_buffer_internal( char *Buffer, const void *DataBegin, const void *DataEnd, @@ -45,7 +46,8 @@ int main(int argc, const char *argv[]) { __llvm_profile_begin_data(), __llvm_profile_end_data(), __llvm_profile_begin_counters(), __llvm_profile_end_counters(), __llvm_profile_begin_bitmap(), __llvm_profile_end_bitmap(), - __llvm_profile_begin_names(), __llvm_profile_end_names()); + __llvm_profile_begin_names(), __llvm_profile_end_names(), NULL, NULL, + NULL, NULL); char *buf = malloc(bufsize); int ret = __llvm_profile_write_buffer_internal( diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h index 1ba577059cc40..25ec06a739202 100644 --- a/llvm/include/llvm/ProfileData/InstrProf.h +++ b/llvm/include/llvm/ProfileData/InstrProf.h @@ -1062,7 +1062,7 @@ struct Header { uint64_t MemProfOffset; uint64_t BinaryIdOffset; uint64_t TemporalProfTracesOffset; - uint64_t VTableNamesOffset; // Organize virtual table names. + uint64_t VTableNamesOffset; // New fields should only be added at the end to ensure that the size // computation is correct. The methods below need to be updated to ensure that // the new field is read correctly. @@ -1207,6 +1207,7 @@ template struct alignas(8) VTableProfileData { #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Type Name; #include "llvm/ProfileData/InstrProfData.inc" }; + // File header structure of the LLVM profile data in raw format. // The definition should match the header referenced in // compiler-rt/lib/profile/InstrProfilingFile.c and diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h index 636f11342887b..cfde5d3fc77d6 100644 --- a/llvm/include/llvm/ProfileData/InstrProfReader.h +++ b/llvm/include/llvm/ProfileData/InstrProfReader.h @@ -664,7 +664,7 @@ class IndexedInstrProfReader : public InstrProfReader { /// When a symtab is constructed from profiles by llvm-profdata, the list of /// names could be decompressed based on `VTableNamePtr` and /// `CompressedVTableNamesLen`. - /// A compiler that reads indexed profiles could construct symtab from module + /// A compiler that reads indexed profiles could construct symtab from module /// IR so it doesn't need the decompressed names. const char *VTableNamePtr = nullptr; /// The length of compressed vtable names. diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 98d5db31b7e7f..6becd38ce7f98 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -367,7 +367,10 @@ TextInstrProfReader::readValueProfileData(InstrProfRecord &Record) { Value = IndexedInstrProf::ComputeHash(VD.first); } } else if (ValueKind == IPVK_VTableTarget) { - // do nothing + if (InstrProfSymtab::isExternalSymbol(VD.first)) + Value = 0; + else + Value = IndexedInstrProf::ComputeHash(VD.first); } else { READ_NUM(VD.first, Value); } diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 79112d6a4631d..08bc0c1aecc4e 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -19,7 +19,6 @@ #include "llvm/ProfileData/InstrProf.h" #include "llvm/ProfileData/MemProf.h" #include "llvm/ProfileData/ProfileCommon.h" -#include "llvm/Support/Compression.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/Error.h" @@ -614,6 +613,10 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) { VTableNamesSectionStart = OS.tell(); + // Use an empty string as compressed vtable names and get the necessary + // profile format change in place for version 12. + // TODO: Store the list of vtable names in InstrProfWriter and use the + // real compressed name. std::string CompressedVTableNames; uint64_t CompressedStringLen = CompressedVTableNames.length(); @@ -626,7 +629,7 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { OS.writeByte(static_cast(c)); // Pad up to a multiple of 8. - // InstrProfReader could read bytes according to 'CompressedStringLen'. + // InstrProfReader would read bytes according to 'CompressedStringLen'. uint64_t PaddedLength = alignTo(CompressedStringLen, 8); for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { From 206415c89f35e16a9ae04199f06f27b95ce1a8d3 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 14 Feb 2024 15:13:22 -0800 Subject: [PATCH 05/13] fix format --- compiler-rt/include/profile/InstrProfData.inc | 3 ++- compiler-rt/lib/profile/InstrProfilingMerge.c | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index fcaa14d6419d0..fbf17b2000e75 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -303,6 +303,7 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #undef COVMAP_HEADER /* COVMAP_HEADER end. */ + #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ @@ -760,7 +761,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" -#define INSTR_PROF_VNAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfvn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" #define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index 33ffd9636dcb7..c6e03db7009a8 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -156,7 +156,6 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, Header->NumCounters * __llvm_profile_counter_entry_size(); SrcBitmapStart = SrcCountersEnd; SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; - SrcValueProfDataStart = SrcNameStart + getDistanceFromCounterToValueProf(Header); if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) From 84570cd9ec08d0a597c0a5df91e3de026436b423 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Thu, 15 Feb 2024 11:39:55 -0800 Subject: [PATCH 06/13] resolve review feedback --- compiler-rt/include/profile/InstrProfData.inc | 2 +- compiler-rt/lib/profile/InstrProfilingMerge.c | 10 ++--- .../llvm/ProfileData/InstrProfData.inc | 2 + llvm/lib/ProfileData/InstrProfWriter.cpp | 45 +++++++++---------- 4 files changed, 28 insertions(+), 31 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index fbf17b2000e75..2dc1a8d6c86b2 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -105,7 +105,7 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #endif INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ - IndexedInstrProf::ComputeHash(PGOVTableName))) + IndexedInstrProf::ComputeHash(PGOVTableName))) INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ VTablePointer, VTableAddr) INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ diff --git a/compiler-rt/lib/profile/InstrProfilingMerge.c b/compiler-rt/lib/profile/InstrProfilingMerge.c index c6e03db7009a8..c0706b73e1668 100644 --- a/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -107,13 +107,13 @@ static uintptr_t signextIfWin64(void *V) { #endif } +// Skip names section, vtable profile data section and vtable names section +// for runtime profile merge. To merge runtime addresses from multiple +// profiles collected from the same instrumented binary, the binary should be +// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR +// disabled). In this set-up these three sections remain unchanged. static uint64_t getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { - // Skip names section, vtable profile data section and vtable names section - // for runtime profile merge. To merge runtime addresses from multiple - // profiles collected from the same instrumented binary, the binary should be - // loaded at fixed base address (e.g., build with -no-pie, or run with ASLR - // disabled). In this set-up these three sections remain unchanged. const uint64_t VTableSectionSize = Header->NumVTables * sizeof(VTableProfData); const uint64_t PaddingBytesAfterVTableSection = diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 8c50006e85038..d5aeff9b9f223 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -331,6 +331,7 @@ INSTR_PROF_SECT_ENTRY(IPSK_covname, \ #undef INSTR_PROF_SECT_ENTRY #endif + #ifdef INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_DATA_DEFINED @@ -502,6 +503,7 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #undef INSTR_PROF_VALUE_PROF_DATA #endif /* INSTR_PROF_VALUE_PROF_DATA */ + #ifdef INSTR_PROF_COMMON_API_IMPL #define INSTR_PROF_DATA_DEFINED #ifdef __cplusplus diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index 08bc0c1aecc4e..bd9b021e57b25 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -458,10 +458,9 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { Header.VTableNamesOffset = 0; int N = sizeof(IndexedInstrProf::Header) / sizeof(uint64_t); - // Only write out all the fields except 'HashOffset', 'MemProfOffset', - // 'BinaryIdOffset', `TemporalProfTracesOffset` and `VTableNamesOffset`. We - // need to remember the offset of these fields to allow back patching later. - for (int I = 0; I < N - 5; I++) + // Only write out the first four fields. We need to remember the offset of the + // remaining fields to allow back patching later. + for (int I = 0; I < 4; I++) OS.write(reinterpret_cast(&Header)[I]); // Save the location of Header.HashOffset field in \c OS. @@ -608,33 +607,29 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { OS.writeByte(0); } - // if version >= the version with vtable profile metadata. - uint64_t VTableNamesSectionStart = 0; - if (IndexedInstrProf::ProfVersion::CurrentVersion >= 12) { - VTableNamesSectionStart = OS.tell(); + uint64_t VTableNamesSectionStart = OS.tell(); - // Use an empty string as compressed vtable names and get the necessary - // profile format change in place for version 12. - // TODO: Store the list of vtable names in InstrProfWriter and use the - // real compressed name. - std::string CompressedVTableNames; + // Use an empty string as compressed vtable names and get the necessary + // profile format change in place for version 12. + // TODO: Store the list of vtable names in InstrProfWriter and use the + // real compressed name. + std::string CompressedVTableNames; - uint64_t CompressedStringLen = CompressedVTableNames.length(); + uint64_t CompressedStringLen = CompressedVTableNames.length(); - // Record the length of compressed string. - OS.write(CompressedStringLen); + // Record the length of compressed string. + OS.write(CompressedStringLen); - // Write the chars in compressed strings. - for (auto &c : CompressedVTableNames) - OS.writeByte(static_cast(c)); + // Write the chars in compressed strings. + for (auto &c : CompressedVTableNames) + OS.writeByte(static_cast(c)); - // Pad up to a multiple of 8. - // InstrProfReader would read bytes according to 'CompressedStringLen'. - uint64_t PaddedLength = alignTo(CompressedStringLen, 8); + // Pad up to a multiple of 8. + // InstrProfReader would read bytes according to 'CompressedStringLen'. + uint64_t PaddedLength = alignTo(CompressedStringLen, 8); - for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { - OS.writeByte(0); - } + for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) { + OS.writeByte(0); } uint64_t TemporalProfTracesSectionStart = 0; From 0c9ed57bce7d29f27ab7562d85503edfa8132670 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Thu, 15 Feb 2024 11:42:07 -0800 Subject: [PATCH 07/13] add back one blank line in llvm/include/llvm/ProfileData/InstrProfData.inc --- llvm/include/llvm/ProfileData/InstrProfData.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index d5aeff9b9f223..d05d8059f0a02 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -286,6 +286,7 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #undef COVMAP_HEADER /* COVMAP_HEADER end. */ + #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ From 67bbafb8642a1fe4e15159377b7f13b0c0755bce Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Thu, 15 Feb 2024 23:21:06 -0800 Subject: [PATCH 08/13] Apply suggestions from code review Co-authored-by: modiking --- compiler-rt/include/profile/InstrProfData.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 2dc1a8d6c86b2..0340f99f15e5e 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -210,17 +210,17 @@ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* For virtual table address profiling, the address point of the virtual table * (i.e., the address contained in objects pointing to a virtual table) are * profiled. Note this may not be the address of the per C++ class virtual table - * object (e.g., there might be an offset). + * object (e.g., there might be an offset). * * The profiled addresses are stored in raw profile, together with the following * two types of information. * 1. The (starting and ending) addresses of per C++ class virtual table objects. * 2. The (compressed) virtual table object names. * RawInstrProfReader converts profiled virtual table addresses to virtual table - * objects' MD5 hash. + * objects' MD5 hash. */ VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., " - "there is an offset from this address to per C++ " + "there is an offset from this address to a C++ " "class virtual table global variable.)") /* These two kinds must be the last to be * declared. This is to make sure the string From c30a6991eae71b15a2f2325a7a0c5244f3b20517 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Thu, 15 Feb 2024 23:24:12 -0800 Subject: [PATCH 09/13] return truncated error if any --- llvm/lib/ProfileData/InstrProfReader.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp index 6becd38ce7f98..31b742bca14d6 100644 --- a/llvm/lib/ProfileData/InstrProfReader.cpp +++ b/llvm/lib/ProfileData/InstrProfReader.cpp @@ -1293,7 +1293,11 @@ Error IndexedInstrProfReader::readHeader() { support::endian::readNext(Ptr); + // Writer first writes the length of compressed string, and then the actual + // content. VTableNamePtr = (const char *)Ptr; + if (VTableNamePtr > (const char *)DataBuffer->getBufferEnd()) + return make_error(instrprof_error::truncated); } if (GET_VERSION(Header->formatVersion()) >= 10 && From 393d166236f8ad45eb516a3ae2b93b8364562b43 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Tue, 20 Feb 2024 16:22:24 -0800 Subject: [PATCH 10/13] resolve review fee feedback --- compiler-rt/include/profile/InstrProfData.inc | 10 +++--- .../lib/profile/InstrProfilingBuffer.c | 24 +++++++++++--- .../llvm/ProfileData/InstrProfData.inc | 31 ++++++++++++++----- llvm/lib/ProfileData/InstrProfWriter.cpp | 6 ++-- 4 files changed, 50 insertions(+), 21 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 0340f99f15e5e..f7d32d2631fa2 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -106,7 +106,7 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ IndexedInstrProf::ComputeHash(PGOVTableName))) -INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ VTablePointer, VTableAddr) INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ @@ -210,18 +210,16 @@ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* For virtual table address profiling, the address point of the virtual table * (i.e., the address contained in objects pointing to a virtual table) are * profiled. Note this may not be the address of the per C++ class virtual table - * object (e.g., there might be an offset). + * object (e.g., there might be an offset). * * The profiled addresses are stored in raw profile, together with the following * two types of information. * 1. The (starting and ending) addresses of per C++ class virtual table objects. * 2. The (compressed) virtual table object names. * RawInstrProfReader converts profiled virtual table addresses to virtual table - * objects' MD5 hash. + * objects' MD5 hash. */ -VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The address of the compatible vtable (i.e., " - "there is an offset from this address to a C++ " - "class virtual table global variable.)") +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index a38d0bb96f11e..9b48980f7f258 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -65,6 +65,15 @@ COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, const __llvm_profile_data *End) { intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + // `sizeof(__llvm_profile_data) - 1` is required in the numerator when + // [Begin, End] represents an inclusive range. + // For ELF, [Begin, End) represents the address of linker-inserted + // symbols `__start__` and `__stop_`. + // Thereby, `End` is one byte past the inclusive range, and + // `sizeof(__llvm_profile_data) - 1` is not necessary in the numerator to get + // the correct number of profile data. + // FIXME: Consider removing `sizeof(__llvm_profile_data) - 1` if this is true + // across platforms. return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); } @@ -75,11 +84,18 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); } +// Counts the number of `VTableProfData` elements within the range of [Begin, +// End). Caller should guarantee that End points to one byte past the inclusive +// range. +// FIXME: Add a compiler-rt test to make sure the number of vtables in the +// raw profile is the same as the number of vtable elements in the instrumented +// binary. COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, const VTableProfData *End) { + // Convert pointers to intptr_t to use integer arithmetic. intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; - return (EndI + sizeof(VTableProfData) - 1 - BeginI) / sizeof(VTableProfData); + return (EndI - BeginI) / sizeof(VTableProfData); } COMPILER_RT_VISIBILITY @@ -241,7 +257,7 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( // Set virtual table arguments to NULL since they are not supported yet. return lprofWriteDataImpl( &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, - BitmapBegin, BitmapEnd, 0 /* VPDataReader */, NamesBegin, NamesEnd, - NULL /* VTableBegin */, NULL /* VTableEnd */, NULL /* VNamesBegin */, - NULL /* VNamesEnd */, 0 /* SkipNameDataWrite */); + BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd, + /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL, + /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0); } diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index d05d8059f0a02..f7d32d2631fa2 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -94,19 +94,22 @@ INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +/* For a virtual table object, record the name hash to associate profiled + * addresses with global variables, and record {starting address, size in bytes} + * to map the profiled virtual table (which usually have an offset from the + * starting address) back to a virtual table object. */ #ifndef INSTR_PROF_VTABLE_DATA #define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) #else #define INSTR_PROF_VTABLE_DATA_DEFINED #endif -INSTR_PROF_VTABLE_DATA( - const uint64_t, llvm::Type::getInt64Ty(Ctx), VTableNameHash, - ConstantInt::get(llvm::Type::getInt64Ty(Ctx), - IndexedInstrProf::ComputeHash(PGOVTableName))) -INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), +INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ + VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ VTablePointer, VTableAddr) -INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, - ConstantInt::get(llvm::Type::getInt32Ty(Ctx), +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ VTableSizeVal)) #undef INSTR_PROF_VTABLE_DATA /* INSTR_PROF_VTABLE_DATA end. */ @@ -204,7 +207,19 @@ VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") -VALUE_PROF_KIND(IPVK_VTableTarget, 2, "vtable target") +/* For virtual table address profiling, the address point of the virtual table + * (i.e., the address contained in objects pointing to a virtual table) are + * profiled. Note this may not be the address of the per C++ class virtual table + * object (e.g., there might be an offset). + * + * The profiled addresses are stored in raw profile, together with the following + * two types of information. + * 1. The (starting and ending) addresses of per C++ class virtual table objects. + * 2. The (compressed) virtual table object names. + * RawInstrProfReader converts profiled virtual table addresses to virtual table + * objects' MD5 hash. + */ +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp index bd9b021e57b25..e5163ebe8ae37 100644 --- a/llvm/lib/ProfileData/InstrProfWriter.cpp +++ b/llvm/lib/ProfileData/InstrProfWriter.cpp @@ -609,11 +609,11 @@ Error InstrProfWriter::writeImpl(ProfOStream &OS) { uint64_t VTableNamesSectionStart = OS.tell(); - // Use an empty string as compressed vtable names and get the necessary - // profile format change in place for version 12. + // Use a dummy (and uncompressed) string as compressed vtable names and get + // the necessary profile format change in place for version 12. // TODO: Store the list of vtable names in InstrProfWriter and use the // real compressed name. - std::string CompressedVTableNames; + std::string CompressedVTableNames = "VTableNames"; uint64_t CompressedStringLen = CompressedVTableNames.length(); From 744b87f2a091fc0bf209f613b7d87ee23406e371 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 21 Feb 2024 09:24:29 -0800 Subject: [PATCH 11/13] resolve review feedback --- compiler-rt/lib/profile/InstrProfilingBuffer.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 9b48980f7f258..0ee514a9f860d 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -93,15 +93,13 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, const VTableProfData *End) { - // Convert pointers to intptr_t to use integer arithmetic. - intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; - return (EndI - BeginI) / sizeof(VTableProfData); + return End - Begin; } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, const VTableProfData *End) { - return __llvm_profile_get_num_vtable(Begin, End) * sizeof(VTableProfData); + return (intptr_t)(End) - (intptr_t)(Begin); } COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { From 36541d943e78341db3c0c717b679e47006ad0610 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 21 Feb 2024 12:28:46 -0800 Subject: [PATCH 12/13] restore __llvm_profile_get_num_vtable, and swap the order of two header fields in raw profile header --- compiler-rt/include/profile/InstrProfData.inc | 2 +- compiler-rt/lib/profile/InstrProfilingBuffer.c | 4 +++- llvm/include/llvm/ProfileData/InstrProfData.inc | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/compiler-rt/include/profile/InstrProfData.inc b/compiler-rt/include/profile/InstrProfData.inc index 48d71c334808f..1f77853bb8baa 100644 --- a/compiler-rt/include/profile/InstrProfData.inc +++ b/compiler-rt/include/profile/InstrProfData.inc @@ -166,8 +166,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) -INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index 0ee514a9f860d..7c5c26f4d113b 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -93,7 +93,9 @@ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, const VTableProfData *End) { - return End - Begin; + // Convert pointers to intptr_t to use integer arithmetic. + intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; + return (EndI - BeginI) / sizeof(VTableProfData); } COMPILER_RT_VISIBILITY diff --git a/llvm/include/llvm/ProfileData/InstrProfData.inc b/llvm/include/llvm/ProfileData/InstrProfData.inc index 48d71c334808f..1f77853bb8baa 100644 --- a/llvm/include/llvm/ProfileData/InstrProfData.inc +++ b/llvm/include/llvm/ProfileData/InstrProfData.inc @@ -166,8 +166,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) -INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ From c74bde70b76f5793ea0be0f972040a7c0ee7f140 Mon Sep 17 00:00:00 2001 From: mingmingl Date: Wed, 21 Feb 2024 14:54:45 -0800 Subject: [PATCH 13/13] llvm/test/tools/llvm-profdata/nocompress.test fails because its raw profile version is outdated. - update llvm/test/tools/llvm-profdata/Inputs/compressed.profraw along with raw profile version bump - The test requires '!zlib', so I generate raw profile with DLLVM_ENABLE_ZLIB=1 and run the test with DLLVM_ENABLE_ZLIB=0. --- .../llvm-profdata/Inputs/compressed.profraw | Bin 1968 -> 1984 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw b/llvm/test/tools/llvm-profdata/Inputs/compressed.profraw index 9966729d92ddc33bf89eeb3fee87215bbabbbef1..e3f77e870d4d20828119348e70eb44e6d39e0ec0 100644 GIT binary patch delta 40 wcmdnMe}JE}u_!ISs37M**F;W#z6X_m+ZL?)|9`G8Q)PVUWItx9jRg+u0BX?@Q2+n{ delta 39 vcmX@Wzk#2#u_!ISs37M*=R{6_L5r?)Gq*SV|KArNnC4N>z`(e%(w!XuMI#TR