diff --git a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h
index 6226135ef84b3..367e6409258f1 100644
--- a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h
@@ -26,10 +26,18 @@ extern "C" {
is not yet freed. */
int __sanitizer_get_ownership(const volatile void *p);
+ /* If a pointer lies within an allocation, it will return the start address
+ of the allocation. Otherwise, it returns nullptr. */
+ const void *__sanitizer_get_allocated_begin(const void *p);
+
/* Returns the number of bytes reserved for the pointer p.
Requires (get_ownership(p) == true) or (p == 0). */
size_t __sanitizer_get_allocated_size(const volatile void *p);
+ /* Returns the number of bytes reserved for the pointer p.
+ Requires __sanitizer_get_allocated_begin(p) == p. */
+ size_t __sanitizer_get_allocated_size_fast(const volatile void *p);
+
/* Number of bytes, allocated and not yet freed by the application. */
size_t __sanitizer_get_current_allocated_bytes(void);
diff --git a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
index 2f415bd9e8540..983df7cea16ed 100644
--- a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
+++ b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h
@@ -129,26 +129,23 @@ int __sanitizer_acquire_crash_state();
/// state mid == end, so that should be the final state when the
/// container is destroyed or when the container reallocates the storage.
///
-/// For ASan, beg should be 8-aligned and end
-/// should be either 8-aligned or it should point to the end of a separate
-/// heap-, stack-, or global-allocated buffer. So the following example will
-/// not work:
+/// For ASan, beg no longer needs to be 8-aligned,
+/// first and last granule may be shared with other objects
+/// and therefore the function can be used for any allocator.
///
-/// \code
-/// int64_t x[2]; // 16 bytes, 8-aligned
-/// char *beg = (char *)&x[0];
-/// char *end = beg + 12; // Not 8-aligned, not the end of the buffer
-/// \endcode
+/// The following example shows how to use the function:
///
-/// The following, however, will work:
/// \code
-/// int32_t x[3]; // 12 bytes, but 8-aligned under ASan.
+/// int32_t x[3]; // 12 bytes
/// char *beg = (char*)&x[0];
-/// char *end = beg + 12; // Not 8-aligned, but is the end of the buffer
+/// char *end = beg + 12;
+/// __sanitizer_annotate_contiguous_container(beg, end, beg, end);
/// \endcode
///
/// \note Use this function with caution and do not use for anything other
/// than vector-like classes.
+/// \note Unaligned beg or end may miss bugs in
+/// these granules.
///
/// \param beg Beginning of memory region.
/// \param end End of memory region.
diff --git a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
index 14035c05c6353..ee742c7f30317 100644
--- a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h
@@ -1,4 +1,4 @@
-//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===//
+//===-- sanitizer/hwasan_interface.h ----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
diff --git a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
index 2782e61fb8c79..f19c79d79ba62 100644
--- a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
+++ b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h
@@ -172,6 +172,129 @@ int __tsan_on_finalize(int failed);
// Release TSan internal memory in a best-effort manner.
void __tsan_flush_memory();
+// User-provided default TSAN options.
+const char* __tsan_default_options(void);
+
+// User-provided default TSAN suppressions.
+const char* __tsan_default_suppressions(void);
+
+/// Returns a report's description.
+///
+/// Returns a report's description (issue type), number of duplicate issues
+/// found, counts of array data (stack traces, memory operations, locations,
+/// mutexes, threads, unique thread IDs) and a stack trace of a sleep()
+/// call (if one was involved in the issue).
+///
+/// \param report Opaque pointer to the current report.
+/// \param[out] description Report type description.
+/// \param[out] count Count of duplicate issues.
+/// \param[out] stack_count Count of stack traces.
+/// \param[out] mop_count Count of memory operations.
+/// \param[out] loc_count Count of locations.
+/// \param[out] mutex_count Count of mutexes.
+/// \param[out] thread_count Count of threads.
+/// \param[out] unique_tid_count Count of unique thread IDs.
+/// \param sleep_trace A buffer to store the stack trace of a sleep()
+/// call.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_data(void *report, const char **description, int *count,
+ int *stack_count, int *mop_count, int *loc_count,
+ int *mutex_count, int *thread_count,
+ int *unique_tid_count, void **sleep_trace,
+ unsigned long trace_size);
+
+/// Returns information about stack traces included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's stacks.
+/// \param trace A buffer to store the stack trace.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_stack(void *report, unsigned long idx, void **trace,
+ unsigned long trace_size);
+
+/// Returns information about memory operations included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's memory operations.
+/// \param[out] tid Thread ID of the memory operation.
+/// \param[out] addr Address of the memory operation.
+/// \param[out] size Size of the memory operation.
+/// \param[out] write Write flag of the memory operation.
+/// \param[out] atomic Atomicity flag of the memory operation.
+/// \param trace A buffer to store the stack trace.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_mop(void *report, unsigned long idx, int *tid,
+ void **addr, int *size, int *write, int *atomic,
+ void **trace, unsigned long trace_size);
+
+/// Returns information about locations included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's locations.
+/// \param[out] type Type of the location.
+/// \param[out] addr Address of the location.
+/// \param[out] start Start of the location.
+/// \param[out] size Size of the location.
+/// \param[out] tid Thread ID of the location.
+/// \param[out] fd File descriptor of the location.
+/// \param[out] suppressable Suppressable flag.
+/// \param trace A buffer to store the stack trace.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_loc(void *report, unsigned long idx, const char **type,
+ void **addr, void **start, unsigned long *size,
+ int *tid, int *fd, int *suppressable, void **trace,
+ unsigned long trace_size);
+
+/// Returns information about mutexes included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's mutexes.
+/// \param[out] mutex_id Id of the mutex.
+/// \param[out] addr Address of the mutex.
+/// \param[out] destroyed Destroyed mutex flag.
+/// \param trace A buffer to store the stack trace.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_mutex(void *report, unsigned long idx, uint64_t *mutex_id,
+ void **addr, int *destroyed, void **trace,
+ unsigned long trace_size);
+
+/// Returns information about threads included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's threads.
+/// \param[out] tid Thread ID of the thread.
+/// \param[out] os_id Operating system's ID of the thread.
+/// \param[out] running Running flag of the thread.
+/// \param[out] name Name of the thread.
+/// \param[out] parent_tid ID of the parent thread.
+/// \param trace A buffer to store the stack trace.
+/// \param trace_size Size in bytes of the trace buffer.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_thread(void *report, unsigned long idx, int *tid,
+ uint64_t *os_id, int *running, const char **name,
+ int *parent_tid, void **trace,
+ unsigned long trace_size);
+
+/// Returns information about unique thread IDs included in the report.
+///
+/// \param report Opaque pointer to the current report.
+/// \param idx Index to the report's unique thread IDs.
+/// \param[out] tid Unique thread ID of the report.
+/// \returns Returns 1 if successful, 0 if not.
+int __tsan_get_report_unique_tid(void *report, unsigned long idx, int *tid);
+
+/// Returns the current report.
+///
+/// If TSan is currently reporting a detected issue on the current thread,
+/// returns an opaque pointer to the current report. Otherwise returns NULL.
+/// \returns An opaque pointer to the current report. Otherwise returns NULL.
+void *__tsan_get_current_report();
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
index 74183fcd24273..15eefcb96063a 100644
--- a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp
@@ -16,6 +16,7 @@
#include "asan_allocator.h"
+#include "asan_internal.h"
#include "asan_mapping.h"
#include "asan_poisoning.h"
#include "asan_report.h"
@@ -24,6 +25,7 @@
#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_allocator_checks.h"
#include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_errno.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
@@ -190,28 +192,56 @@ class LargeChunkHeader {
}
};
+static void FillChunk(AsanChunk *m) {
+ // FIXME: Use ReleaseMemoryPagesToOS.
+ Flags &fl = *flags();
+
+ if (fl.max_free_fill_size > 0) {
+ // We have to skip the chunk header, it contains free_context_id.
+ uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
+ if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area.
+ uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
+ size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size);
+ REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill);
+ }
+ }
+}
+
struct QuarantineCallback {
QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack)
: cache_(cache),
stack_(stack) {
}
- void Recycle(AsanChunk *m) {
+ void PreQuarantine(AsanChunk *m) const {
+ FillChunk(m);
+ // Poison the region.
+ PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
+ kAsanHeapFreeMagic);
+ }
+
+ void Recycle(AsanChunk *m) const {
void *p = get_allocator().GetBlockBegin(m);
- if (p != m) {
- // Clear the magic value, as allocator internals may overwrite the
- // contents of deallocated chunk, confusing GetAsanChunk lookup.
- reinterpret_cast(p)->Set(nullptr);
- }
- u8 old_chunk_state = CHUNK_QUARANTINE;
- if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
- CHUNK_INVALID, memory_order_acquire)) {
- CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
- }
+ // The secondary will immediately unpoison and unmap the memory, so this
+ // branch is unnecessary.
+ if (get_allocator().FromPrimary(p)) {
+ if (p != m) {
+ // Clear the magic value, as allocator internals may overwrite the
+ // contents of deallocated chunk, confusing GetAsanChunk lookup.
+ reinterpret_cast(p)->Set(nullptr);
+ }
- PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
- kAsanHeapLeftRedzoneMagic);
+ u8 old_chunk_state = CHUNK_QUARANTINE;
+ if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state,
+ CHUNK_INVALID,
+ memory_order_acquire)) {
+ CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE);
+ }
+
+ PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
+ kAsanHeapLeftRedzoneMagic);
+ }
// Statistics.
AsanStats &thread_stats = GetCurrentThreadStats();
@@ -221,7 +251,17 @@ struct QuarantineCallback {
get_allocator().Deallocate(cache_, p);
}
- void *Allocate(uptr size) {
+ void RecyclePassThrough(AsanChunk *m) const {
+ // Recycle for the secondary will immediately unpoison and unmap the
+ // memory, so quarantine preparation is unnecessary.
+ if (get_allocator().FromPrimary(m)) {
+ // The primary allocation may need pattern fill if enabled.
+ FillChunk(m);
+ }
+ Recycle(m);
+ }
+
+ void *Allocate(uptr size) const {
void *res = get_allocator().Allocate(cache_, size, 1);
// TODO(alekseys): Consider making quarantine OOM-friendly.
if (UNLIKELY(!res))
@@ -229,9 +269,7 @@ struct QuarantineCallback {
return res;
}
- void Deallocate(void *p) {
- get_allocator().Deallocate(cache_, p);
- }
+ void Deallocate(void *p) const { get_allocator().Deallocate(cache_, p); }
private:
AllocatorCache* const cache_;
@@ -248,6 +286,22 @@ void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const {
thread_stats.mmaps++;
thread_stats.mmaped += size;
}
+
+void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin,
+ uptr user_size) const {
+ uptr user_end = RoundDownTo(user_begin + user_size, ASAN_SHADOW_GRANULARITY);
+ user_begin = RoundUpTo(user_begin, ASAN_SHADOW_GRANULARITY);
+ // The secondary mapping will be immediately returned to user, no value
+ // poisoning that with non-zero just before unpoisoning by Allocate(). So just
+ // poison head/tail invisible to Allocate().
+ PoisonShadow(p, user_begin - p, kAsanHeapLeftRedzoneMagic);
+ PoisonShadow(user_end, size - (user_end - p), kAsanHeapLeftRedzoneMagic);
+ // Statistics.
+ AsanStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.mmaps++;
+ thread_stats.mmaped += size;
+}
+
void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const {
PoisonShadow(p, size, 0);
// We are about to unmap a chunk of user memory.
@@ -387,8 +441,9 @@ struct Allocator {
}
void GetOptions(AllocatorOptions *options) const {
- options->quarantine_size_mb = quarantine.GetSize() >> 20;
- options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10;
+ options->quarantine_size_mb = quarantine.GetMaxSize() >> 20;
+ options->thread_local_quarantine_size_kb =
+ quarantine.GetMaxCacheSize() >> 10;
options->min_redzone = atomic_load(&min_redzone, memory_order_acquire);
options->max_redzone = atomic_load(&max_redzone, memory_order_acquire);
options->may_return_null = AllocatorMayReturnNull();
@@ -502,9 +557,10 @@ struct Allocator {
uptr needed_size = rounded_size + rz_size;
if (alignment > min_alignment)
needed_size += alignment;
+ bool from_primary = PrimaryAllocator::CanAllocate(needed_size, alignment);
// If we are allocating from the secondary allocator, there will be no
// automatic right redzone, so add the right redzone manually.
- if (!PrimaryAllocator::CanAllocate(needed_size, alignment))
+ if (!from_primary)
needed_size += rz_size;
CHECK(IsAligned(needed_size, min_alignment));
if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize ||
@@ -536,15 +592,6 @@ struct Allocator {
ReportOutOfMemory(size, stack);
}
- if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) {
- // Heap poisoning is enabled, but the allocator provides an unpoisoned
- // chunk. This is possible if CanPoisonMemory() was false for some
- // time, for example, due to flags()->start_disabled.
- // Anyway, poison the block before using it for anything else.
- uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated);
- PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic);
- }
-
uptr alloc_beg = reinterpret_cast(allocated);
uptr alloc_end = alloc_beg + needed_size;
uptr user_beg = alloc_beg + rz_size;
@@ -561,6 +608,17 @@ struct Allocator {
m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack));
+ if (!from_primary || *(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0) {
+ // The allocator provides an unpoisoned chunk. This is possible for the
+ // secondary allocator, or if CanPoisonMemory() was false for some time,
+ // for example, due to flags()->start_disabled. Anyway, poison left and
+ // right of the block before using it for anything else.
+ uptr tail_beg = RoundUpTo(user_end, ASAN_SHADOW_GRANULARITY);
+ uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(allocated);
+ PoisonShadow(alloc_beg, user_beg - alloc_beg, kAsanHeapLeftRedzoneMagic);
+ PoisonShadow(tail_beg, tail_end - tail_beg, kAsanHeapLeftRedzoneMagic);
+ }
+
uptr size_rounded_down_to_granularity =
RoundDownTo(size, ASAN_SHADOW_GRANULARITY);
// Unpoison the bulk of the memory region.
@@ -628,25 +686,6 @@ struct Allocator {
AsanThread *t = GetCurrentThread();
m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack));
- Flags &fl = *flags();
- if (fl.max_free_fill_size > 0) {
- // We have to skip the chunk header, it contains free_context_id.
- uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size;
- if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area.
- uptr size_to_fill = m->UsedSize() - kChunkHeader2Size;
- size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size);
- REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill);
- }
- }
-
- // Poison the region.
- PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY),
- kAsanHeapFreeMagic);
-
- AsanStats &thread_stats = GetCurrentThreadStats();
- thread_stats.frees++;
- thread_stats.freed += m->UsedSize();
-
// Push into quarantine.
if (t) {
AsanThreadLocalMallocStorage *ms = &t->malloc_storage();
@@ -699,6 +738,10 @@ struct Allocator {
}
}
+ AsanStats &thread_stats = GetCurrentThreadStats();
+ thread_stats.frees++;
+ thread_stats.freed += m->UsedSize();
+
QuarantineChunk(m, ptr, stack);
}
@@ -798,6 +841,10 @@ struct Allocator {
return m->UsedSize();
}
+ uptr AllocationSizeFast(uptr p) {
+ return reinterpret_cast(p - kChunkHeaderSize)->UsedSize();
+ }
+
AsanChunkView FindHeapChunkByAddress(uptr addr) {
AsanChunk *m1 = GetAsanChunkByAddr(addr);
sptr offset = 0;
@@ -1100,6 +1147,10 @@ uptr GetUserBegin(uptr chunk) {
return m ? m->Beg() : 0;
}
+uptr GetUserAddr(uptr chunk) {
+ return chunk;
+}
+
LsanMetadata::LsanMetadata(uptr chunk) {
metadata_ = chunk ? reinterpret_cast(chunk - __asan::kChunkHeaderSize)
: nullptr;
@@ -1140,7 +1191,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) {
__asan::get_allocator().ForEachChunk(callback, arg);
}
-IgnoreObjectResult IgnoreObjectLocked(const void *p) {
+IgnoreObjectResult IgnoreObject(const void *p) {
uptr addr = reinterpret_cast(p);
__asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr);
if (!m ||
@@ -1160,6 +1211,17 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) {
// ---------------------- Interface ---------------- {{{1
using namespace __asan;
+static const void *AllocationBegin(const void *p) {
+ AsanChunk *m = __asan::instance.GetAsanChunkByAddr((uptr)p);
+ if (!m)
+ return nullptr;
+ if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED)
+ return nullptr;
+ if (m->UsedSize() == 0)
+ return nullptr;
+ return (const void *)(m->Beg());
+}
+
// ASan allocator doesn't reserve extra bytes, so normally we would
// just return "size". We don't want to expose our redzone sizes, etc here.
uptr __sanitizer_get_estimated_allocated_size(uptr size) {
@@ -1183,6 +1245,17 @@ uptr __sanitizer_get_allocated_size(const void *p) {
return allocated_size;
}
+uptr __sanitizer_get_allocated_size_fast(const void *p) {
+ DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
+ uptr ret = instance.AllocationSizeFast(reinterpret_cast(p));
+ DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
+ return ret;
+}
+
+const void *__sanitizer_get_allocated_begin(const void *p) {
+ return AllocationBegin(p);
+}
+
void __sanitizer_purge_allocator() {
GET_STACK_TRACE_MALLOC;
instance.Purge(&stack);
diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.h b/system/lib/compiler-rt/lib/asan/asan_allocator.h
index 0b4dbf03bb9d5..ffeedadf0772d 100644
--- a/system/lib/compiler-rt/lib/asan/asan_allocator.h
+++ b/system/lib/compiler-rt/lib/asan/asan_allocator.h
@@ -114,6 +114,7 @@ class AsanChunkFifoList: public IntrusiveList {
struct AsanMapUnmapCallback {
void OnMap(uptr p, uptr size) const;
+ void OnMapSecondary(uptr p, uptr size, uptr user_begin, uptr user_size) const;
void OnUnmap(uptr p, uptr size) const;
};
@@ -143,11 +144,15 @@ typedef DefaultSizeClassMap SizeClassMap;
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x8000000000ULL; // 500G
typedef DefaultSizeClassMap SizeClassMap;
-# else
+# elif SANITIZER_APPLE
const uptr kAllocatorSpace = 0x600000000000ULL;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
typedef DefaultSizeClassMap SizeClassMap;
-# endif
+# else
+const uptr kAllocatorSpace = 0x500000000000ULL;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+typedef DefaultSizeClassMap SizeClassMap;
+# endif
template
struct AP64 { // Allocator64 parameters. Deliberately using a short name.
static const uptr kSpaceBeg = kAllocatorSpace;
diff --git a/system/lib/compiler-rt/lib/asan/asan_globals.cpp b/system/lib/compiler-rt/lib/asan/asan_globals.cpp
index 80de7403a4bc3..a358c9cadf985 100644
--- a/system/lib/compiler-rt/lib/asan/asan_globals.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_globals.cpp
@@ -92,6 +92,10 @@ static void ReportGlobal(const Global &g, const char *prefix) {
if (info.line != 0) {
Report(" location: name=%s, %d\n", info.file, static_cast(info.line));
}
+ else if (g.gcc_location != 0) {
+ // Fallback to Global::gcc_location
+ Report(" location: name=%s, %d\n", g.gcc_location->filename, g.gcc_location->line_no);
+ }
}
static u32 FindRegistrationSite(const Global *g) {
@@ -302,6 +306,11 @@ void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g) {
if (info.line != 0) {
str->append("%s:%d", info.file, static_cast(info.line));
+ } else if (g.gcc_location != 0) {
+ // Fallback to Global::gcc_location
+ str->append("%s", g.gcc_location->filename ? g.gcc_location->filename : g.module_name);
+ if (g.gcc_location->line_no) str->append(":%d", g.gcc_location->line_no);
+ if (g.gcc_location->column_no) str->append(":%d", g.gcc_location->column_no);
} else {
str->append("%s", g.module_name);
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
index 2f0ef224e3d7e..bd87c64d78e96 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "asan_interceptors.h"
+
#include "asan_allocator.h"
#include "asan_internal.h"
#include "asan_mapping.h"
@@ -20,7 +21,10 @@
#include "asan_stack.h"
#include "asan_stats.h"
#include "asan_suppressions.h"
+#include "asan_thread.h"
#include "lsan/lsan_common.h"
+#include "sanitizer_common/sanitizer_errno.h"
+#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_libc.h"
// There is no general interception at all on Fuchsia or Emscripten.
@@ -84,12 +88,6 @@ using namespace __asan;
DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr)
DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
-#define ASAN_INTERCEPTOR_ENTER(ctx, func) \
- AsanInterceptorContext _ctx = {#func}; \
- ctx = (void *)&_ctx; \
- (void) ctx; \
-
-#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
#define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
ASAN_INTERCEPT_FUNC_VER(name, ver)
#define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
@@ -149,22 +147,46 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
*begin = *end = 0; \
}
-#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
- do { \
- ASAN_INTERCEPTOR_ENTER(ctx, memmove); \
- ASAN_MEMMOVE_IMPL(ctx, to, from, size); \
- } while (false)
+template
+static void* mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length,
+ int prot, int flags, int fd, OFF64_T offset) {
+ void *res = real_mmap(addr, length, prot, flags, fd, offset);
+ if (length && res != (void *)-1) {
+ const uptr beg = reinterpret_cast(res);
+ DCHECK(IsAligned(beg, GetPageSize()));
+ SIZE_T rounded_length = RoundUpTo(length, GetPageSize());
+ // Only unpoison shadow if it's an ASAN managed address.
+ if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1))
+ PoisonShadow(beg, RoundUpTo(length, GetPageSize()), 0);
+ }
+ return res;
+}
-#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
- do { \
- ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \
- ASAN_MEMCPY_IMPL(ctx, to, from, size); \
+template
+static int munmap_interceptor(Munmap real_munmap, void *addr, SIZE_T length) {
+ // We should not tag if munmap fail, but it's to late to tag after
+ // real_munmap, as the pages could be mmaped by another thread.
+ const uptr beg = reinterpret_cast(addr);
+ if (length && IsAligned(beg, GetPageSize())) {
+ SIZE_T rounded_length = RoundUpTo(length, GetPageSize());
+ // Protect from unmapping the shadow.
+ if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1))
+ PoisonShadow(beg, rounded_length, 0);
+ }
+ return real_munmap(addr, length);
+}
+
+# define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, \
+ fd, offset) \
+ do { \
+ (void)(ctx); \
+ return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off); \
} while (false)
-#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
- do { \
- ASAN_INTERCEPTOR_ENTER(ctx, memset); \
- ASAN_MEMSET_IMPL(ctx, block, c, size); \
+# define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, length) \
+ do { \
+ (void)(ctx); \
+ return munmap_interceptor(REAL(munmap), addr, sz); \
} while (false)
#if CAN_SANITIZE_LEAKS
@@ -172,6 +194,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
__lsan::ScopedInterceptorDisabler disabler
#endif
+#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_ASAN_INITED()
+
#include "sanitizer_common/sanitizer_common_interceptors.inc"
#include "sanitizer_common/sanitizer_signal_interceptors.inc"
@@ -196,19 +220,26 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) {
AsanThread *t = (AsanThread *)arg;
SetCurrentThread(t);
- return t->ThreadStart(GetTid());
+ auto self = GetThreadSelf();
+ auto args = asanThreadArgRetval().GetArgs(self);
+ thread_return_t retval = t->ThreadStart(GetTid());
+ asanThreadArgRetval().Finish(self, retval);
+ CHECK_EQ(args.arg_retval, t->get_arg());
+ return retval;
}
-INTERCEPTOR(int, pthread_create, void *thread,
- void *attr, void *(*start_routine)(void*), void *arg) {
+INTERCEPTOR(int, pthread_create, void *thread, void *attr,
+ void *(*start_routine)(void *), void *arg) {
EnsureMainThreadIDIsCorrect();
// Strict init-order checking is thread-hostile.
if (flags()->strict_init_order)
StopInitOrderChecking();
GET_STACK_TRACE_THREAD;
- int detached = 0;
- if (attr)
- REAL(pthread_attr_getdetachstate)(attr, &detached);
+ bool detached = [attr]() {
+ int d = 0;
+ return attr && !REAL(pthread_attr_getdetachstate)(attr, &d) &&
+ IsStateDetached(d);
+ }();
u32 current_tid = GetCurrentTidOrInvalid();
AsanThread *t =
@@ -220,10 +251,13 @@ INTERCEPTOR(int, pthread_create, void *thread,
// stored by pthread for future reuse even after thread destruction, and
// the linked list it's stored in doesn't even hold valid pointers to the
// objects, the latter are calculated by obscure pointer arithmetic.
-#if CAN_SANITIZE_LEAKS
+# if CAN_SANITIZE_LEAKS
__lsan::ScopedInterceptorDisabler disabler;
-#endif
- result = REAL(pthread_create)(thread, attr, asan_thread_start, t);
+# endif
+ asanThreadArgRetval().Create(detached, {start_routine, arg}, [&]() -> uptr {
+ result = REAL(pthread_create)(thread, attr, asan_thread_start, t);
+ return result ? 0 : *(uptr *)(thread);
+ });
}
if (result != 0) {
// If the thread didn't start delete the AsanThread to avoid leaking it.
@@ -234,9 +268,51 @@ INTERCEPTOR(int, pthread_create, void *thread,
return result;
}
-INTERCEPTOR(int, pthread_join, void *t, void **arg) {
- return real_pthread_join(t, arg);
+INTERCEPTOR(int, pthread_join, void *thread, void **retval) {
+ int result;
+ asanThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_join)(thread, retval);
+ return !result;
+ });
+ return result;
+}
+
+INTERCEPTOR(int, pthread_detach, void *thread) {
+ int result;
+ asanThreadArgRetval().Detach((uptr)thread, [&]() {
+ result = REAL(pthread_detach)(thread);
+ return !result;
+ });
+ return result;
+}
+
+INTERCEPTOR(void, pthread_exit, void *retval) {
+ asanThreadArgRetval().Finish(GetThreadSelf(), retval);
+ REAL(pthread_exit)(retval);
+}
+
+# if ASAN_INTERCEPT_TRYJOIN
+INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) {
+ int result;
+ asanThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_tryjoin_np)(thread, ret);
+ return !result;
+ });
+ return result;
+}
+# endif
+
+# if ASAN_INTERCEPT_TIMEDJOIN
+INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret,
+ const struct timespec *abstime) {
+ int result;
+ asanThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_timedjoin_np)(thread, ret, abstime);
+ return !result;
+ });
+ return result;
}
+# endif
DEFINE_REAL_PTHREAD_FUNCTIONS
#endif // ASAN_INTERCEPT_PTHREAD_CREATE
@@ -388,7 +464,7 @@ INTERCEPTOR(_Unwind_Reason_Code, _Unwind_SjLj_RaiseException,
#if ASAN_INTERCEPT_INDEX
# if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX
INTERCEPTOR(char*, index, const char *string, int c)
- ALIAS(WRAPPER_NAME(strchr));
+ ALIAS(WRAP(strchr));
# else
# if SANITIZER_APPLE
DECLARE_REAL(char*, index, const char *string, int c)
@@ -474,7 +550,9 @@ INTERCEPTOR(char*, strdup, const char *s) {
}
GET_STACK_TRACE_MALLOC;
void *new_mem = asan_malloc(length + 1, &stack);
- REAL(memcpy)(new_mem, s, length + 1);
+ if (new_mem) {
+ REAL(memcpy)(new_mem, s, length + 1);
+ }
return reinterpret_cast(new_mem);
}
@@ -490,7 +568,9 @@ INTERCEPTOR(char*, __strdup, const char *s) {
}
GET_STACK_TRACE_MALLOC;
void *new_mem = asan_malloc(length + 1, &stack);
- REAL(memcpy)(new_mem, s, length + 1);
+ if (new_mem) {
+ REAL(memcpy)(new_mem, s, length + 1);
+ }
return reinterpret_cast(new_mem);
}
#endif // ASAN_INTERCEPT___STRDUP
@@ -508,19 +588,34 @@ INTERCEPTOR(char*, strncpy, char *to, const char *from, uptr size) {
return REAL(strncpy)(to, from, size);
}
-INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) {
- void *ctx;
- ASAN_INTERCEPTOR_ENTER(ctx, strtol);
- ENSURE_ASAN_INITED();
- if (!flags()->replace_str) {
- return REAL(strtol)(nptr, endptr, base);
- }
+template
+static ALWAYS_INLINE auto StrtolImpl(void *ctx, Fn real, const char *nptr,
+ char **endptr, int base)
+ -> decltype(real(nullptr, nullptr, 0)) {
+ if (!flags()->replace_str)
+ return real(nptr, endptr, base);
char *real_endptr;
- long result = REAL(strtol)(nptr, &real_endptr, base);
+ auto res = real(nptr, &real_endptr, base);
StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
- return result;
+ return res;
}
+# define INTERCEPTOR_STRTO_BASE(ret_type, func) \
+ INTERCEPTOR(ret_type, func, const char *nptr, char **endptr, int base) { \
+ void *ctx; \
+ ASAN_INTERCEPTOR_ENTER(ctx, func); \
+ ENSURE_ASAN_INITED(); \
+ return StrtolImpl(ctx, REAL(func), nptr, endptr, base); \
+ }
+
+INTERCEPTOR_STRTO_BASE(long, strtol)
+INTERCEPTOR_STRTO_BASE(long long, strtoll)
+
+# if SANITIZER_GLIBC
+INTERCEPTOR_STRTO_BASE(long, __isoc23_strtol)
+INTERCEPTOR_STRTO_BASE(long long, __isoc23_strtoll)
+# endif
+
INTERCEPTOR(int, atoi, const char *nptr) {
void *ctx;
ASAN_INTERCEPTOR_ENTER(ctx, atoi);
@@ -559,20 +654,6 @@ INTERCEPTOR(long, atol, const char *nptr) {
return result;
}
-#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL
-INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) {
- void *ctx;
- ASAN_INTERCEPTOR_ENTER(ctx, strtoll);
- ENSURE_ASAN_INITED();
- if (!flags()->replace_str) {
- return REAL(strtoll)(nptr, endptr, base);
- }
- char *real_endptr;
- long long result = REAL(strtoll)(nptr, &real_endptr, base);
- StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
- return result;
-}
-
INTERCEPTOR(long long, atoll, const char *nptr) {
void *ctx;
ASAN_INTERCEPTOR_ENTER(ctx, atoll);
@@ -586,7 +667,6 @@ INTERCEPTOR(long long, atoll, const char *nptr) {
ASAN_READ_STRING(ctx, nptr, (real_endptr - nptr) + 1);
return result;
}
-#endif // ASAN_INTERCEPT_ATOLL_AND_STRTOLL
#if ASAN_INTERCEPT___CXA_ATEXIT || ASAN_INTERCEPT_ATEXIT
static void AtCxaAtexit(void *unused) {
@@ -652,6 +732,7 @@ void InitializeAsanInterceptors() {
static bool was_called_once;
CHECK(!was_called_once);
was_called_once = true;
+ InitializePlatformInterceptors();
InitializeCommonInterceptors();
InitializeSignalInterceptors();
@@ -670,11 +751,13 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(atoi);
ASAN_INTERCEPT_FUNC(atol);
- ASAN_INTERCEPT_FUNC(strtol);
-#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL
ASAN_INTERCEPT_FUNC(atoll);
+ ASAN_INTERCEPT_FUNC(strtol);
ASAN_INTERCEPT_FUNC(strtoll);
-#endif
+# if SANITIZER_GLIBC
+ ASAN_INTERCEPT_FUNC(__isoc23_strtol);
+ ASAN_INTERCEPT_FUNC(__isoc23_strtoll);
+# endif
// Intecept jump-related functions.
ASAN_INTERCEPT_FUNC(longjmp);
@@ -702,11 +785,11 @@ void InitializeAsanInterceptors() {
#endif
// Indirectly intercept std::rethrow_exception.
#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION
- INTERCEPT_FUNCTION(_Unwind_RaiseException);
+ ASAN_INTERCEPT_FUNC(_Unwind_RaiseException);
#endif
// Indirectly intercept std::rethrow_exception.
#if ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION
- INTERCEPT_FUNCTION(_Unwind_SjLj_RaiseException);
+ ASAN_INTERCEPT_FUNC(_Unwind_SjLj_RaiseException);
#endif
// Intercept threading-related functions
@@ -718,6 +801,16 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(pthread_create);
#endif
ASAN_INTERCEPT_FUNC(pthread_join);
+ ASAN_INTERCEPT_FUNC(pthread_detach);
+ ASAN_INTERCEPT_FUNC(pthread_exit);
+# endif
+
+# if ASAN_INTERCEPT_TIMEDJOIN
+ ASAN_INTERCEPT_FUNC(pthread_timedjoin_np);
+#endif
+
+#if ASAN_INTERCEPT_TRYJOIN
+ ASAN_INTERCEPT_FUNC(pthread_tryjoin_np);
#endif
// Intercept atexit function.
@@ -737,8 +830,6 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(vfork);
#endif
- InitializePlatformInterceptors();
-
VReport(1, "AddressSanitizer: libc interceptors initialized\n");
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.h b/system/lib/compiler-rt/lib/asan/asan_interceptors.h
index c4bf087ea17f0..d00d05587b368 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors.h
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.h
@@ -42,12 +42,10 @@ void InitializePlatformInterceptors();
// Use macro to describe if specific function should be
// intercepted on a given platform.
#if !SANITIZER_WINDOWS
-# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 1
# define ASAN_INTERCEPT__LONGJMP 1
# define ASAN_INTERCEPT_INDEX 1
# define ASAN_INTERCEPT_PTHREAD_CREATE 1
#else
-# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 0
# define ASAN_INTERCEPT__LONGJMP 0
# define ASAN_INTERCEPT_INDEX 0
# define ASAN_INTERCEPT_PTHREAD_CREATE 0
@@ -78,8 +76,8 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT___LONGJMP_CHK 0
#endif
-#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \
- !SANITIZER_NETBSD
+#if ASAN_HAS_EXCEPTIONS && !SANITIZER_SOLARIS && !SANITIZER_NETBSD && \
+ (!SANITIZER_WINDOWS || (defined(__MINGW32__) && defined(__i386__)))
# define ASAN_INTERCEPT___CXA_THROW 1
# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1
# if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__))
@@ -112,6 +110,14 @@ void InitializePlatformInterceptors();
# define ASAN_INTERCEPT___STRDUP 0
#endif
+#if SANITIZER_GLIBC && ASAN_INTERCEPT_PTHREAD_CREATE
+# define ASAN_INTERCEPT_TIMEDJOIN 1
+# define ASAN_INTERCEPT_TRYJOIN 1
+#else
+# define ASAN_INTERCEPT_TIMEDJOIN 0
+# define ASAN_INTERCEPT_TRYJOIN 0
+#endif
+
#if SANITIZER_LINUX && \
(defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \
defined(__x86_64__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64)
@@ -158,6 +164,12 @@ DECLARE_REAL(char*, strstr, const char *s1, const char *s2)
# define ASAN_INTERCEPT_FUNC(name)
# endif // SANITIZER_APPLE
+#define ASAN_INTERCEPTOR_ENTER(ctx, func) \
+ AsanInterceptorContext _ctx = {#func}; \
+ ctx = (void *)&_ctx; \
+ (void) ctx;
+#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name)
+
#endif // !SANITIZER_FUCHSIA
#endif // ASAN_INTERCEPTORS_H
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp
index 9e275ba211e2f..7c12a0f5810ab 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp
@@ -11,13 +11,54 @@
// ASan versions of memcpy, memmove, and memset.
//===---------------------------------------------------------------------===//
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
#include "asan_interceptors_memintrinsics.h"
+
+#include "asan_interceptors.h"
#include "asan_report.h"
#include "asan_stack.h"
#include "asan_suppressions.h"
using namespace __asan;
+// memcpy is called during __asan_init() from the internals of printf(...).
+// We do not treat memcpy with to==from as a bug.
+// See http://llvm.org/bugs/show_bug.cgi?id=11763.
+#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ if (LIKELY(to != from)) { \
+ CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
+ } \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } else if (UNLIKELY(!asan_inited)) { \
+ return internal_memcpy(to, from, size); \
+ } \
+ return REAL(memcpy)(to, from, size); \
+ } while (0)
+
+// memset is called inside Printf.
+#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ ASAN_WRITE_RANGE(ctx, block, size); \
+ } else if (UNLIKELY(!asan_inited)) { \
+ return internal_memset(block, c, size); \
+ } \
+ return REAL(memset)(block, c, size); \
+ } while (0)
+
+#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
+ do { \
+ if (LIKELY(replace_intrin_cached)) { \
+ ASAN_READ_RANGE(ctx, from, size); \
+ ASAN_WRITE_RANGE(ctx, to, size); \
+ } \
+ return internal_memmove(to, from, size); \
+ } while (0)
+
void *__asan_memcpy(void *to, const void *from, uptr size) {
ASAN_MEMCPY_IMPL(nullptr, to, from, size);
}
@@ -40,4 +81,26 @@ extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]];
extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]];
extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]];
+#else // SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
+
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \
+ do { \
+ ASAN_INTERCEPTOR_ENTER(ctx, memmove); \
+ ASAN_MEMMOVE_IMPL(ctx, to, from, size); \
+ } while (false)
+
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \
+ do { \
+ ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \
+ ASAN_MEMCPY_IMPL(ctx, to, from, size); \
+ } while (false)
+
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \
+ do { \
+ ASAN_INTERCEPTOR_ENTER(ctx, memset); \
+ ASAN_MEMSET_IMPL(ctx, block, c, size); \
+ } while (false)
+
+#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc"
+
#endif // SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
index bbc5390ceaa40..eb44f8f2f729b 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
+++ b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h
@@ -79,43 +79,6 @@ struct AsanInterceptorContext {
} \
} while (0)
-// memcpy is called during __asan_init() from the internals of printf(...).
-// We do not treat memcpy with to==from as a bug.
-// See http://llvm.org/bugs/show_bug.cgi?id=11763.
-#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \
- do { \
- if (LIKELY(replace_intrin_cached)) { \
- if (LIKELY(to != from)) { \
- CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \
- } \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } else if (UNLIKELY(!asan_inited)) { \
- return internal_memcpy(to, from, size); \
- } \
- return REAL(memcpy)(to, from, size); \
- } while (0)
-
-// memset is called inside Printf.
-#define ASAN_MEMSET_IMPL(ctx, block, c, size) \
- do { \
- if (LIKELY(replace_intrin_cached)) { \
- ASAN_WRITE_RANGE(ctx, block, size); \
- } else if (UNLIKELY(!asan_inited)) { \
- return internal_memset(block, c, size); \
- } \
- return REAL(memset)(block, c, size); \
- } while (0)
-
-#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \
- do { \
- if (LIKELY(replace_intrin_cached)) { \
- ASAN_READ_RANGE(ctx, from, size); \
- ASAN_WRITE_RANGE(ctx, to, size); \
- } \
- return internal_memmove(to, from, size); \
- } while (0)
-
#define ASAN_READ_RANGE(ctx, offset, size) \
ACCESS_MEMORY_RANGE(ctx, offset, size, false)
#define ASAN_WRITE_RANGE(ctx, offset, size) \
diff --git a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
index 987f855c0f9ce..a998263780221 100644
--- a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
+++ b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h
@@ -53,9 +53,10 @@ extern "C" {
const char *module_name; // Module name as a C string. This pointer is a
// unique identifier of a module.
uptr has_dynamic_init; // Non-zero if the global has dynamic initializer.
- uptr windows_padding; // TODO: Figure out how to remove this padding
- // that's simply here to make the MSVC incremental
- // linker happy...
+ __asan_global_source_location *gcc_location; // Source location of a global,
+ // used by GCC compiler. LLVM uses
+ // llvm-symbolizer that relies
+ // on DWARF debugging info.
uptr odr_indicator; // The address of the ODR indicator symbol.
};
diff --git a/system/lib/compiler-rt/lib/asan/asan_mac.cpp b/system/lib/compiler-rt/lib/asan/asan_mac.cpp
index c9bd5fb8e1a8f..0c07500614478 100644
--- a/system/lib/compiler-rt/lib/asan/asan_mac.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_mac.cpp
@@ -130,6 +130,18 @@ typedef void* dispatch_source_t;
typedef u64 dispatch_time_t;
typedef void (*dispatch_function_t)(void *block);
typedef void* (*worker_t)(void *block);
+typedef unsigned long dispatch_mach_reason;
+typedef void *dispatch_mach_msg_t;
+typedef int mach_error_t;
+typedef void *dispatch_mach_t;
+
+typedef void (*dispatch_mach_handler_function_t)(void *context,
+ dispatch_mach_reason reason,
+ dispatch_mach_msg_t message,
+ mach_error_t error);
+typedef void (^dispatch_mach_handler_t)(dispatch_mach_reason reason,
+ dispatch_mach_msg_t message,
+ mach_error_t error);
// A wrapper for the ObjC blocks used to support libdispatch.
typedef struct {
@@ -160,7 +172,7 @@ void asan_dispatch_call_block_and_release(void *block) {
VReport(2,
"asan_dispatch_call_block_and_release(): "
"context: %p, pthread_self: %p\n",
- block, pthread_self());
+ block, (void*)pthread_self());
asan_register_worker_thread(context->parent_tid, &stack);
// Call the original dispatcher for the block.
context->func(context->block);
@@ -193,7 +205,7 @@ asan_block_context_t *alloc_asan_context(void *ctxt, dispatch_function_t func,
asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); \
if (Verbosity() >= 2) { \
Report(#dispatch_x_f "(): context: %p, pthread_self: %p\n", \
- asan_ctxt, pthread_self()); \
+ (void*)asan_ctxt, (void*)pthread_self()); \
PRINT_CURRENT_STACK(); \
} \
return REAL(dispatch_x_f)(dq, (void*)asan_ctxt, \
@@ -210,7 +222,7 @@ INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when,
GET_STACK_TRACE_THREAD;
asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack);
if (Verbosity() >= 2) {
- Report("dispatch_after_f: %p\n", asan_ctxt);
+ Report("dispatch_after_f: %p\n", (void*)asan_ctxt);
PRINT_CURRENT_STACK();
}
return REAL(dispatch_after_f)(when, dq, (void*)asan_ctxt,
@@ -224,7 +236,7 @@ INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack);
if (Verbosity() >= 2) {
Report("dispatch_group_async_f(): context: %p, pthread_self: %p\n",
- asan_ctxt, pthread_self());
+ (void*)asan_ctxt, (void*)pthread_self());
PRINT_CURRENT_STACK();
}
REAL(dispatch_group_async_f)(group, dq, (void*)asan_ctxt,
@@ -241,6 +253,8 @@ void dispatch_after(dispatch_time_t when, dispatch_queue_t queue,
void dispatch_source_set_cancel_handler(dispatch_source_t ds,
void(^work)(void));
void dispatch_source_set_event_handler(dispatch_source_t ds, void(^work)(void));
+dispatch_mach_t dispatch_mach_create(const char *label, dispatch_queue_t queue,
+ dispatch_mach_handler_t handler);
}
#define GET_ASAN_BLOCK(work) \
@@ -290,6 +304,34 @@ INTERCEPTOR(void, dispatch_source_set_event_handler,
GET_ASAN_BLOCK(work);
REAL(dispatch_source_set_event_handler)(ds, asan_block);
}
+
+INTERCEPTOR(void *, dispatch_mach_create, const char *label,
+ dispatch_queue_t dq, dispatch_mach_handler_t handler) {
+ int parent_tid = GetCurrentTidOrInvalid();
+ return REAL(dispatch_mach_create)(
+ label, dq,
+ ^(dispatch_mach_reason reason, dispatch_mach_msg_t message,
+ mach_error_t error) {
+ GET_STACK_TRACE_THREAD;
+ asan_register_worker_thread(parent_tid, &stack);
+ handler(reason, message, error);
+ });
+}
+
+INTERCEPTOR(void *, dispatch_mach_create_f, const char *label,
+ dispatch_queue_t dq, void *ctxt,
+ dispatch_mach_handler_function_t handler) {
+ int parent_tid = GetCurrentTidOrInvalid();
+ return REAL(dispatch_mach_create)(
+ label, dq,
+ ^(dispatch_mach_reason reason, dispatch_mach_msg_t message,
+ mach_error_t error) {
+ GET_STACK_TRACE_THREAD;
+ asan_register_worker_thread(parent_tid, &stack);
+ handler(ctxt, reason, message, error);
+ });
+}
+
#endif
#endif // SANITIZER_APPLE
diff --git a/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp b/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp
index 4b76d4ebd3eb4..ff78d7646a90d 100644
--- a/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp
@@ -508,10 +508,10 @@ void ReplaceSystemMalloc() {
TryToOverrideFunction("_expand_base", (uptr)_expand);
if (flags()->windows_hook_rtl_allocators) {
- INTERCEPT_FUNCTION(HeapSize);
- INTERCEPT_FUNCTION(HeapFree);
- INTERCEPT_FUNCTION(HeapReAlloc);
- INTERCEPT_FUNCTION(HeapAlloc);
+ ASAN_INTERCEPT_FUNC(HeapSize);
+ ASAN_INTERCEPT_FUNC(HeapFree);
+ ASAN_INTERCEPT_FUNC(HeapReAlloc);
+ ASAN_INTERCEPT_FUNC(HeapAlloc);
// Undocumented functions must be intercepted by name, not by symbol.
__interception::OverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap),
diff --git a/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp b/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp
index 4fcd5600ed1a1..3396fc2bab94b 100644
--- a/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp
@@ -11,12 +11,11 @@
// This file implements __sanitizer_print_memory_profile.
//===----------------------------------------------------------------------===//
+#include "asan/asan_allocator.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_stacktrace.h"
-#include "sanitizer_common/sanitizer_stoptheworld.h"
-#include "lsan/lsan_common.h"
-#include "asan/asan_allocator.h"
#if CAN_SANITIZE_LEAKS
@@ -100,17 +99,16 @@ static void ChunkCallback(uptr chunk, void *arg) {
FindHeapChunkByAllocBeg(chunk));
}
-static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list,
- void *argument) {
+static void MemoryProfileCB(uptr top_percent, uptr max_number_of_contexts) {
HeapProfile hp;
+ __lsan::LockAllocator();
__lsan::ForEachChunk(ChunkCallback, &hp);
- uptr *Arg = reinterpret_cast(argument);
- hp.Print(Arg[0], Arg[1]);
+ __lsan::UnlockAllocator();
+ hp.Print(top_percent, max_number_of_contexts);
if (Verbosity())
__asan_print_accumulated_stats();
}
-
} // namespace __asan
#endif // CAN_SANITIZE_LEAKS
@@ -120,10 +118,7 @@ SANITIZER_INTERFACE_ATTRIBUTE
void __sanitizer_print_memory_profile(uptr top_percent,
uptr max_number_of_contexts) {
#if CAN_SANITIZE_LEAKS
- uptr Arg[2];
- Arg[0] = top_percent;
- Arg[1] = max_number_of_contexts;
- __sanitizer::StopTheWorld(__asan::MemoryProfileCB, Arg);
+ __asan::MemoryProfileCB(top_percent, max_number_of_contexts);
#endif // CAN_SANITIZE_LEAKS
}
} // extern "C"
diff --git a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
index e1ff872067ff9..4f9daf23ef7ca 100644
--- a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp
@@ -458,11 +458,11 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p,
// FIXME: Two of these three checks are disabled until we fix
// https://github.com/google/sanitizers/issues/258.
// if (d1 != d2)
- // CHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1);
+ // DCHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1);
if (a + granularity <= d1)
- CHECK_EQ(*(u8 *)MemToShadow(a), 0);
+ DCHECK_EQ(*(u8 *)MemToShadow(a), 0);
// if (d2 + granularity <= c && c <= end)
- // CHECK_EQ(*(u8 *)MemToShadow(c - granularity),
+ // DCHECK_EQ(*(u8 *)MemToShadow(c - granularity),
// kAsanContiguousContainerOOBMagic);
uptr b1 = RoundDownTo(new_end, granularity);
diff --git a/system/lib/compiler-rt/lib/asan/asan_stack.h b/system/lib/compiler-rt/lib/asan/asan_stack.h
index b9575d2f427ee..02a76af847ae6 100644
--- a/system/lib/compiler-rt/lib/asan/asan_stack.h
+++ b/system/lib/compiler-rt/lib/asan/asan_stack.h
@@ -32,24 +32,24 @@ u32 GetMallocContextSize();
// as early as possible (in functions exposed to the user), as we generally
// don't want stack trace to contain functions from ASan internals.
-#define GET_STACK_TRACE(max_size, fast) \
- BufferedStackTrace stack; \
- if (max_size <= 2) { \
- stack.size = max_size; \
- if (max_size > 0) { \
- stack.top_frame_bp = GET_CURRENT_FRAME(); \
- stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \
- if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \
- } \
- } else { \
- stack.Unwind(StackTrace::GetCurrentPc(), \
- GET_CURRENT_FRAME(), nullptr, fast, max_size); \
+#define GET_STACK_TRACE(max_size, fast) \
+ UNINITIALIZED BufferedStackTrace stack; \
+ if (max_size <= 2) { \
+ stack.size = max_size; \
+ if (max_size > 0) { \
+ stack.top_frame_bp = GET_CURRENT_FRAME(); \
+ stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \
+ if (max_size > 1) \
+ stack.trace_buffer[1] = GET_CALLER_PC(); \
+ } \
+ } else { \
+ stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \
+ fast, max_size); \
}
-#define GET_STACK_TRACE_FATAL(pc, bp) \
- BufferedStackTrace stack; \
- stack.Unwind(pc, bp, nullptr, \
- common_flags()->fast_unwind_on_fatal)
+#define GET_STACK_TRACE_FATAL(pc, bp) \
+ UNINITIALIZED BufferedStackTrace stack; \
+ stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal)
#define GET_STACK_TRACE_FATAL_HERE \
GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal)
diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.cpp b/system/lib/compiler-rt/lib/asan/asan_thread.cpp
index c991c085a3fd9..df5ef82036c68 100644
--- a/system/lib/compiler-rt/lib/asan/asan_thread.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_thread.cpp
@@ -10,24 +10,25 @@
//
// Thread-related code.
//===----------------------------------------------------------------------===//
+#include "asan_thread.h"
+
#include "asan_allocator.h"
#include "asan_interceptors.h"
+#include "asan_mapping.h"
#include "asan_poisoning.h"
#include "asan_stack.h"
-#include "asan_thread.h"
-#include "asan_mapping.h"
+#include "lsan/lsan_common.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_placement_new.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_tls_get_addr.h"
-#include "lsan/lsan_common.h"
namespace __asan {
// AsanThreadContext implementation.
void AsanThreadContext::OnCreated(void *arg) {
- CreateThreadContextArgs *args = static_cast(arg);
+ CreateThreadContextArgs *args = static_cast(arg);
if (args->stack)
stack_id = StackDepotPut(*args->stack);
thread = args->thread;
@@ -39,34 +40,50 @@ void AsanThreadContext::OnFinished() {
thread = nullptr;
}
-// MIPS requires aligned address
-static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)];
static ThreadRegistry *asan_thread_registry;
+static ThreadArgRetval *thread_data;
static Mutex mu_for_thread_context;
static LowLevelAllocator allocator_for_thread_context;
static ThreadContextBase *GetAsanThreadContext(u32 tid) {
Lock lock(&mu_for_thread_context);
- return new(allocator_for_thread_context) AsanThreadContext(tid);
+ return new (allocator_for_thread_context) AsanThreadContext(tid);
}
-ThreadRegistry &asanThreadRegistry() {
+static void InitThreads() {
static bool initialized;
// Don't worry about thread_safety - this should be called when there is
// a single thread.
- if (!initialized) {
- // Never reuse ASan threads: we store pointer to AsanThreadContext
- // in TSD and can't reliably tell when no more TSD destructors will
- // be called. It would be wrong to reuse AsanThreadContext for another
- // thread before all TSD destructors will be called for it.
- asan_thread_registry =
- new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext);
- initialized = true;
- }
+ if (LIKELY(initialized))
+ return;
+ // Never reuse ASan threads: we store pointer to AsanThreadContext
+ // in TSD and can't reliably tell when no more TSD destructors will
+ // be called. It would be wrong to reuse AsanThreadContext for another
+ // thread before all TSD destructors will be called for it.
+
+ // MIPS requires aligned address
+ static ALIGNED(alignof(
+ ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)];
+ static ALIGNED(alignof(
+ ThreadArgRetval)) char thread_data_placeholder[sizeof(ThreadArgRetval)];
+
+ asan_thread_registry =
+ new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext);
+ thread_data = new (thread_data_placeholder) ThreadArgRetval();
+ initialized = true;
+}
+
+ThreadRegistry &asanThreadRegistry() {
+ InitThreads();
return *asan_thread_registry;
}
+ThreadArgRetval &asanThreadArgRetval() {
+ InitThreads();
+ return *thread_data;
+}
+
AsanThreadContext *GetThreadContextByTidLocked(u32 tid) {
return static_cast(
asanThreadRegistry().GetThreadLocked(tid));
@@ -79,7 +96,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg,
bool detached) {
uptr PageSize = GetPageSizeCached();
uptr size = RoundUpTo(sizeof(AsanThread), PageSize);
- AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__);
+ AsanThread *thread = (AsanThread *)MmapOrDie(size, __func__);
thread->start_routine_ = start_routine;
thread->arg_ = arg;
AsanThreadContext::CreateThreadContextArgs args = {thread, stack};
@@ -89,7 +106,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg,
}
void AsanThread::TSDDtor(void *tsd) {
- AsanThreadContext *context = (AsanThreadContext*)tsd;
+ AsanThreadContext *context = (AsanThreadContext *)tsd;
VReport(1, "T%d TSDDtor\n", context->tid);
if (context->thread)
context->thread->Destroy();
@@ -145,8 +162,7 @@ void AsanThread::StartSwitchFiber(FakeStack **fake_stack_save, uptr bottom,
current_fake_stack->Destroy(this->tid());
}
-void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save,
- uptr *bottom_old,
+void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old,
uptr *size_old) {
if (!atomic_load(&stack_switching_, memory_order_relaxed)) {
Report("ERROR: finishing a fiber switch that has not started\n");
@@ -172,7 +188,8 @@ void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save,
inline AsanThread::StackBounds AsanThread::GetStackBounds() const {
if (!atomic_load(&stack_switching_, memory_order_acquire)) {
// Make sure the stack bounds are fully initialized.
- if (stack_bottom_ >= stack_top_) return {0, 0};
+ if (stack_bottom_ >= stack_top_)
+ return {0, 0};
return {stack_bottom_, stack_top_};
}
char local;
@@ -185,13 +202,9 @@ inline AsanThread::StackBounds AsanThread::GetStackBounds() const {
return {stack_bottom_, stack_top_};
}
-uptr AsanThread::stack_top() {
- return GetStackBounds().top;
-}
+uptr AsanThread::stack_top() { return GetStackBounds().top; }
-uptr AsanThread::stack_bottom() {
- return GetStackBounds().bottom;
-}
+uptr AsanThread::stack_bottom() { return GetStackBounds().bottom; }
uptr AsanThread::stack_size() {
const auto bounds = GetStackBounds();
@@ -212,8 +225,8 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() {
// This CAS checks if the state was 0 and if so changes it to state 1,
// if that was successful, it initializes the pointer.
if (atomic_compare_exchange_strong(
- reinterpret_cast(&fake_stack_), &old_val, 1UL,
- memory_order_relaxed)) {
+ reinterpret_cast(&fake_stack_), &old_val, 1UL,
+ memory_order_relaxed)) {
uptr stack_size_log = Log2(RoundUpToPowerOfTwo(stack_size));
CHECK_LE(flags()->min_uar_stack_size_log, flags()->max_uar_stack_size_log);
stack_size_log =
@@ -267,7 +280,8 @@ thread_return_t AsanThread::ThreadStart(tid_t os_id) {
asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr);
#if !SANITIZER_EMSCRIPTEN
- if (common_flags()->use_sigaltstack) SetAlternateSignalStack();
+ if (common_flags()->use_sigaltstack)
+ SetAlternateSignalStack();
#endif
if (!start_routine_) {
@@ -310,6 +324,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) {
GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size,
&tls_begin_, &tls_size);
stack_top_ = RoundDownTo(stack_bottom_ + stack_size, ASAN_SHADOW_GRANULARITY);
+ stack_bottom_ = RoundDownTo(stack_bottom_, ASAN_SHADOW_GRANULARITY);
tls_end_ = tls_begin_ + tls_size;
dtls_ = DTLS_Get();
@@ -343,14 +358,14 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
bottom = fake_stack->AddrIsInFakeStack(addr);
CHECK(bottom);
access->offset = addr - bottom;
- access->frame_pc = ((uptr*)bottom)[2];
- access->frame_descr = (const char *)((uptr*)bottom)[1];
+ access->frame_pc = ((uptr *)bottom)[2];
+ access->frame_descr = (const char *)((uptr *)bottom)[1];
return true;
}
uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr.
uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY);
- u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
- u8 *shadow_bottom = (u8*)MemToShadow(bottom);
+ u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr);
+ u8 *shadow_bottom = (u8 *)MemToShadow(bottom);
while (shadow_ptr >= shadow_bottom &&
*shadow_ptr != kAsanStackLeftRedzoneMagic) {
@@ -372,7 +387,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr,
CHECK(ptr[0] == kCurrentStackFrameMagic);
access->offset = addr - (uptr)ptr;
access->frame_pc = ptr[2];
- access->frame_descr = (const char*)ptr[1];
+ access->frame_descr = (const char *)ptr[1];
return true;
}
@@ -390,8 +405,8 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) {
}
uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr.
- u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr);
- u8 *shadow_bottom = (u8*)MemToShadow(bottom);
+ u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr);
+ u8 *shadow_bottom = (u8 *)MemToShadow(bottom);
while (shadow_ptr >= shadow_bottom &&
(*shadow_ptr != kAsanStackLeftRedzoneMagic &&
@@ -475,16 +490,23 @@ void EnsureMainThreadIDIsCorrect() {
__asan::AsanThread *GetAsanThreadByOsIDLocked(tid_t os_id) {
__asan::AsanThreadContext *context = static_cast<__asan::AsanThreadContext *>(
__asan::asanThreadRegistry().FindThreadContextByOsIDLocked(os_id));
- if (!context) return nullptr;
+ if (!context)
+ return nullptr;
return context->thread;
}
-} // namespace __asan
+} // namespace __asan
// --- Implementation of LSan-specific functions --- {{{1
namespace __lsan {
-void LockThreadRegistry() { __asan::asanThreadRegistry().Lock(); }
+void LockThreads() {
+ __asan::asanThreadRegistry().Lock();
+ __asan::asanThreadArgRetval().Lock();
+}
-void UnlockThreadRegistry() { __asan::asanThreadRegistry().Unlock(); }
+void UnlockThreads() {
+ __asan::asanThreadArgRetval().Unlock();
+ __asan::asanThreadRegistry().Unlock();
+}
static ThreadRegistry *GetAsanThreadRegistryLocked() {
__asan::asanThreadRegistry().CheckLocked();
@@ -497,7 +519,8 @@ bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end,
uptr *tls_begin, uptr *tls_end, uptr *cache_begin,
uptr *cache_end, DTLS **dtls) {
__asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id);
- if (!t) return false;
+ if (!t)
+ return false;
*stack_begin = t->stack_bottom();
*stack_end = t->stack_top();
*tls_begin = t->tls_begin();
@@ -538,33 +561,7 @@ void GetThreadExtraStackRangesLocked(InternalMmapVector *ranges) {
}
void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) {
- GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked(
- [](ThreadContextBase *tctx, void *ptrs) {
- // Look for the arg pointer of threads that have been created or are
- // running. This is necessary to prevent false positive leaks due to the
- // AsanThread holding the only live reference to a heap object. This
- // can happen because the `pthread_create()` interceptor doesn't wait
- // for the child thread to start before returning and thus loosing the
- // the only live reference to the heap object on the stack.
-
- __asan::AsanThreadContext *atctx =
- static_cast<__asan::AsanThreadContext *>(tctx);
-
- // Note ThreadStatusRunning is required because there is a small window
- // where the thread status switches to `ThreadStatusRunning` but the
- // `arg` pointer still isn't on the stack yet.
- if (atctx->status != ThreadStatusCreated &&
- atctx->status != ThreadStatusRunning)
- return;
-
- uptr thread_arg = reinterpret_cast(atctx->thread->get_arg());
- if (!thread_arg)
- return;
-
- auto ptrsVec = reinterpret_cast *>(ptrs);
- ptrsVec->push_back(thread_arg);
- },
- ptrs);
+ __asan::asanThreadArgRetval().GetAllPtrsLocked(ptrs);
}
void GetRunningThreadsLocked(InternalMmapVector *threads) {
@@ -577,11 +574,7 @@ void GetRunningThreadsLocked(InternalMmapVector *threads) {
threads);
}
-void FinishThreadLocked(u32 tid) {
- GetAsanThreadRegistryLocked()->FinishThread(tid);
-}
-
-} // namespace __lsan
+} // namespace __lsan
namespace __sanitizer {
ThreadRegistry *GetThreadRegistryLocked() {
@@ -601,20 +594,18 @@ void __sanitizer_start_switch_fiber(void **fakestacksave, const void *bottom,
VReport(1, "__asan_start_switch_fiber called from unknown thread\n");
return;
}
- t->StartSwitchFiber((FakeStack**)fakestacksave, (uptr)bottom, size);
+ t->StartSwitchFiber((FakeStack **)fakestacksave, (uptr)bottom, size);
}
SANITIZER_INTERFACE_ATTRIBUTE
-void __sanitizer_finish_switch_fiber(void* fakestack,
- const void **bottom_old,
+void __sanitizer_finish_switch_fiber(void *fakestack, const void **bottom_old,
uptr *size_old) {
AsanThread *t = GetCurrentThread();
if (!t) {
VReport(1, "__asan_finish_switch_fiber called from unknown thread\n");
return;
}
- t->FinishSwitchFiber((FakeStack*)fakestack,
- (uptr*)bottom_old,
- (uptr*)size_old);
+ t->FinishSwitchFiber((FakeStack *)fakestack, (uptr *)bottom_old,
+ (uptr *)size_old);
}
}
diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.h b/system/lib/compiler-rt/lib/asan/asan_thread.h
index 801a3960ec6cb..c131dd40d8647 100644
--- a/system/lib/compiler-rt/lib/asan/asan_thread.h
+++ b/system/lib/compiler-rt/lib/asan/asan_thread.h
@@ -15,11 +15,12 @@
#define ASAN_THREAD_H
#include "asan_allocator.h"
-#include "asan_internal.h"
#include "asan_fake_stack.h"
+#include "asan_internal.h"
#include "asan_stats.h"
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_libc.h"
+#include "sanitizer_common/sanitizer_thread_arg_retval.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
namespace __sanitizer {
@@ -129,7 +130,7 @@ class AsanThread {
void *extra_spill_area() { return &extra_spill_area_; }
- void *get_arg() { return arg_; }
+ void *get_arg() const { return arg_; }
private:
// NOTE: There is no AsanThread constructor. It is allocated
@@ -171,6 +172,7 @@ class AsanThread {
// Returns a single instance of registry.
ThreadRegistry &asanThreadRegistry();
+ThreadArgRetval &asanThreadArgRetval();
// Must be called under ThreadRegistryLock.
AsanThreadContext *GetThreadContextByTidLocked(u32 tid);
diff --git a/system/lib/compiler-rt/lib/asan/asan_win.cpp b/system/lib/compiler-rt/lib/asan/asan_win.cpp
index 7dbd7ab98a17b..25f2e6cd551fe 100644
--- a/system/lib/compiler-rt/lib/asan/asan_win.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_win.cpp
@@ -159,6 +159,8 @@ INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security,
namespace __asan {
void InitializePlatformInterceptors() {
+ __interception::SetErrorReportCallback(Report);
+
// The interceptors were not designed to be removable, so we have to keep this
// module alive for the life of the process.
HMODULE pinned;
@@ -194,9 +196,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
}
void FlushUnneededASanShadowMemory(uptr p, uptr size) {
+ // Only asan on 64-bit Windows supports committing shadow memory on demand.
+#if SANITIZER_WINDOWS64
// Since asan's mapping is compacting, the shadow chunk may be
// not page-aligned, so we only flush the page-aligned portion.
ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size));
+#endif
}
// ---------------------- TSD ---------------- {{{
diff --git a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp
index e3a90f18ed81a..0fa636bec0d00 100644
--- a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp
+++ b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp
@@ -65,6 +65,7 @@ INTERCEPT_WRAP_W_W(_expand_dbg)
INTERCEPT_LIBRARY_FUNCTION(atoi);
INTERCEPT_LIBRARY_FUNCTION(atol);
+INTERCEPT_LIBRARY_FUNCTION(atoll);
INTERCEPT_LIBRARY_FUNCTION(frexp);
INTERCEPT_LIBRARY_FUNCTION(longjmp);
#if SANITIZER_INTERCEPT_MEMCHR
@@ -91,6 +92,7 @@ INTERCEPT_LIBRARY_FUNCTION(strspn);
INTERCEPT_LIBRARY_FUNCTION(strstr);
INTERCEPT_LIBRARY_FUNCTION(strtok);
INTERCEPT_LIBRARY_FUNCTION(strtol);
+INTERCEPT_LIBRARY_FUNCTION(strtoll);
INTERCEPT_LIBRARY_FUNCTION(wcslen);
INTERCEPT_LIBRARY_FUNCTION(wcsnlen);
diff --git a/system/lib/compiler-rt/lib/builtins/absvdi2.c b/system/lib/compiler-rt/lib/builtins/absvdi2.c
index b9566cd874fe4..291ab5f7f91d5 100644
--- a/system/lib/compiler-rt/lib/builtins/absvdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/absvdi2.c
@@ -18,7 +18,7 @@
COMPILER_RT_ABI di_int __absvdi2(di_int a) {
const int N = (int)(sizeof(di_int) * CHAR_BIT);
- if (a == ((di_int)1 << (N - 1)))
+ if (a == ((di_int)((du_int)1 << (N - 1))))
compilerrt_abort();
const di_int t = a >> (N - 1);
return (a ^ t) - t;
diff --git a/system/lib/compiler-rt/lib/builtins/absvsi2.c b/system/lib/compiler-rt/lib/builtins/absvsi2.c
index 9d5de7e8a3f22..9977c33d8f7ea 100644
--- a/system/lib/compiler-rt/lib/builtins/absvsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/absvsi2.c
@@ -18,7 +18,7 @@
COMPILER_RT_ABI si_int __absvsi2(si_int a) {
const int N = (int)(sizeof(si_int) * CHAR_BIT);
- if (a == ((si_int)1 << (N - 1)))
+ if (a == ((si_int)((su_int)1 << (N - 1))))
compilerrt_abort();
const si_int t = a >> (N - 1);
return (a ^ t) - t;
diff --git a/system/lib/compiler-rt/lib/builtins/addtf3.c b/system/lib/compiler-rt/lib/builtins/addtf3.c
index 86e4f4cfc3fc6..2cb3a4d591917 100644
--- a/system/lib/compiler-rt/lib/builtins/addtf3.c
+++ b/system/lib/compiler-rt/lib/builtins/addtf3.c
@@ -13,7 +13,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#include "fp_add_impl.inc"
COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b) {
diff --git a/system/lib/compiler-rt/lib/builtins/ashlti3.c b/system/lib/compiler-rt/lib/builtins/ashlti3.c
index 2d7bd4a893809..99a133ffa22f8 100644
--- a/system/lib/compiler-rt/lib/builtins/ashlti3.c
+++ b/system/lib/compiler-rt/lib/builtins/ashlti3.c
@@ -18,7 +18,7 @@
// Precondition: 0 <= b < bits_in_tword
-COMPILER_RT_ABI ti_int __ashlti3(ti_int a, si_int b) {
+COMPILER_RT_ABI ti_int __ashlti3(ti_int a, int b) {
const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT);
twords input;
twords result;
diff --git a/system/lib/compiler-rt/lib/builtins/ashrti3.c b/system/lib/compiler-rt/lib/builtins/ashrti3.c
index f573b6d6ccba4..b306051df0287 100644
--- a/system/lib/compiler-rt/lib/builtins/ashrti3.c
+++ b/system/lib/compiler-rt/lib/builtins/ashrti3.c
@@ -18,7 +18,7 @@
// Precondition: 0 <= b < bits_in_tword
-COMPILER_RT_ABI ti_int __ashrti3(ti_int a, si_int b) {
+COMPILER_RT_ABI ti_int __ashrti3(ti_int a, int b) {
const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT);
twords input;
twords result;
diff --git a/system/lib/compiler-rt/lib/builtins/assembly.h b/system/lib/compiler-rt/lib/builtins/assembly.h
index 69a3d8620f924..169d49683f503 100644
--- a/system/lib/compiler-rt/lib/builtins/assembly.h
+++ b/system/lib/compiler-rt/lib/builtins/assembly.h
@@ -267,7 +267,7 @@
#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
.globl SYMBOL_NAME(name) SEPARATOR \
SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \
- DECLARE_SYMBOL_VISIBILITY(SYMBOL_NAME(name)) SEPARATOR \
+ DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \
.set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR
#if defined(__ARM_EABI__)
diff --git a/system/lib/compiler-rt/lib/builtins/clear_cache.c b/system/lib/compiler-rt/lib/builtins/clear_cache.c
index 8993761eb3d42..2ac99b25c243f 100644
--- a/system/lib/compiler-rt/lib/builtins/clear_cache.c
+++ b/system/lib/compiler-rt/lib/builtins/clear_cache.c
@@ -110,10 +110,14 @@ void __clear_cache(void *start, void *end) {
"jr.hb $at\n"
"move $at, $0\n"
".set at");
-#else
+#elif defined(__linux__) || defined(__OpenBSD__)
// Pre-R6 may not be globalized. And some implementations may give strange
// synci_step. So, let's use libc call for it.
- cacheflush(start, end_int - start_int, BCACHE);
+ _flush_cache(start, end_int - start_int, BCACHE);
+#else
+ (void)start_int;
+ (void)end_int;
+ compilerrt_abort();
#endif
}
#elif defined(__aarch64__) && !defined(__APPLE__)
diff --git a/system/lib/compiler-rt/lib/builtins/comparetf2.c b/system/lib/compiler-rt/lib/builtins/comparetf2.c
index f1592454138cd..be5e9e5e44dd2 100644
--- a/system/lib/compiler-rt/lib/builtins/comparetf2.c
+++ b/system/lib/compiler-rt/lib/builtins/comparetf2.c
@@ -39,7 +39,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#include "fp_compare_impl.inc"
COMPILER_RT_ABI CMP_RESULT __letf2(fp_t a, fp_t b) { return __leXf2__(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/cpu_model.c b/system/lib/compiler-rt/lib/builtins/cpu_model.c
index f5ad530c7e886..0bd7296fb2526 100644
--- a/system/lib/compiler-rt/lib/builtins/cpu_model.c
+++ b/system/lib/compiler-rt/lib/builtins/cpu_model.c
@@ -113,6 +113,7 @@ enum ProcessorSubtypes {
ZHAOXIN_FAM7H_LUJIAZUI,
AMDFAM19H_ZNVER4,
INTEL_COREI7_GRANITERAPIDS,
+ INTEL_COREI7_GRANITERAPIDS_D,
CPU_SUBTYPE_MAX
};
@@ -474,13 +475,19 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
break;
// Granite Rapids:
- case 0xae:
case 0xad:
CPU = "graniterapids";
*Type = INTEL_COREI7;
*Subtype = INTEL_COREI7_GRANITERAPIDS;
break;
+ // Granite Rapids D:
+ case 0xae:
+ CPU = "graniterapids-d";
+ *Type = INTEL_COREI7;
+ *Subtype = INTEL_COREI7_GRANITERAPIDS_D;
+ break;
+
case 0x1c: // Most 45 nm Intel Atom processors
case 0x26: // 45 nm Atom Lincroft
case 0x27: // 32 nm Atom Medfield
@@ -646,7 +653,7 @@ getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
unsigned *Features) {
- unsigned EAX, EBX;
+ unsigned EAX = 0, EBX = 0;
#define setFeature(F) \
Features[F / 32] |= 1U << (F % 32)
@@ -744,8 +751,11 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save)
setFeature(FEATURE_AVX512VP2INTERSECT);
+ // EAX from subleaf 0 is the maximum subleaf supported. Some CPUs don't
+ // return all 0s for invalid subleaves so check the limit.
bool HasLeaf7Subleaf1 =
- MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
+ HasLeaf7 && EAX >= 1 &&
+ !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save)
setFeature(FEATURE_AVX512BF16);
@@ -839,6 +849,39 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
}
#elif defined(__aarch64__)
+// LSE support detection for out-of-line atomics
+// using HWCAP and Auxiliary vector
+_Bool __aarch64_have_lse_atomics
+ __attribute__((visibility("hidden"), nocommon));
+
+#if defined(__has_include)
+#if __has_include()
+#include
+
+#if __has_include()
+#include
+#else
+typedef struct __ifunc_arg_t {
+ unsigned long _size;
+ unsigned long _hwcap;
+ unsigned long _hwcap2;
+} __ifunc_arg_t;
+#endif // __has_include()
+
+#if __has_include()
+#include
+
+#if defined(__ANDROID__)
+#include
+#include
+#elif defined(__Fuchsia__)
+#include
+#include
+#endif
+
+#ifndef _IFUNC_ARG_HWCAP
+#define _IFUNC_ARG_HWCAP (1ULL << 62)
+#endif
#ifndef AT_HWCAP
#define AT_HWCAP 16
#endif
@@ -1009,25 +1052,6 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
#define HWCAP2_SVE_EBF16 (1UL << 33)
#endif
-// LSE support detection for out-of-line atomics
-// using HWCAP and Auxiliary vector
-_Bool __aarch64_have_lse_atomics
- __attribute__((visibility("hidden"), nocommon));
-
-#if defined(__has_include)
-#if __has_include()
-#include
-#if __has_include()
-#include
-
-#if defined(__ANDROID__)
-#include
-#include
-#elif defined(__Fuchsia__)
-#include
-#include
-#endif
-
// Detect Exynos 9810 CPU
#define IF_EXYNOS9810 \
char arch[PROP_VALUE_MAX]; \
@@ -1140,11 +1164,16 @@ struct {
// As features grows new fields could be added
} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
-void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) {
+void init_cpu_features_resolver(unsigned long hwcap, const __ifunc_arg_t *arg) {
#define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
#define extractBits(val, start, number) \
(val & ((1ULL << number) - 1ULL) << start) >> start
+ if (__aarch64_cpu_features.features)
+ return;
+ unsigned long hwcap2 = 0;
+ if (hwcap & _IFUNC_ARG_HWCAP)
+ hwcap2 = arg->_hwcap2;
if (hwcap & HWCAP_CRC32)
setCPUFeature(FEAT_CRC);
if (hwcap & HWCAP_PMULL)
@@ -1320,6 +1349,7 @@ void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) {
if (hwcap & HWCAP_SHA3)
setCPUFeature(FEAT_SHA3);
}
+ setCPUFeature(FEAT_MAX);
}
void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) {
@@ -1328,7 +1358,6 @@ void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) {
// CPU features already initialized.
if (__aarch64_cpu_features.features)
return;
- setCPUFeature(FEAT_MAX);
#if defined(__FreeBSD__)
int res = 0;
res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
@@ -1344,7 +1373,11 @@ void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) {
hwcap = getauxval(AT_HWCAP);
hwcap2 = getauxval(AT_HWCAP2);
#endif // defined(__FreeBSD__)
- init_cpu_features_resolver(hwcap, hwcap2);
+ __ifunc_arg_t arg;
+ arg._size = sizeof(__ifunc_arg_t);
+ arg._hwcap = hwcap;
+ arg._hwcap2 = hwcap2;
+ init_cpu_features_resolver(hwcap | _IFUNC_ARG_HWCAP, &arg);
#undef extractBits
#undef getCPUFeature
#undef setCPUFeature
diff --git a/system/lib/compiler-rt/lib/builtins/crtbegin.c b/system/lib/compiler-rt/lib/builtins/crtbegin.c
new file mode 100644
index 0000000000000..a0860ca12ea03
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/crtbegin.c
@@ -0,0 +1,135 @@
+//===-- crtbegin.c - Start of constructors and destructors ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include
+
+__attribute__((visibility("hidden"))) void *__dso_handle = &__dso_handle;
+
+#ifdef EH_USE_FRAME_REGISTRY
+__extension__ static void *__EH_FRAME_LIST__[]
+ __attribute__((section(".eh_frame"), aligned(sizeof(void *)))) = {};
+
+extern void __register_frame_info(const void *, void *) __attribute__((weak));
+extern void *__deregister_frame_info(const void *) __attribute__((weak));
+#endif
+
+#ifndef CRT_HAS_INITFINI_ARRAY
+typedef void (*fp)(void);
+
+static fp __CTOR_LIST__[]
+ __attribute__((section(".ctors"), aligned(sizeof(fp)))) = {(fp)-1};
+extern fp __CTOR_LIST_END__[];
+#endif
+
+extern void __cxa_finalize(void *) __attribute__((weak));
+
+static void __attribute__((used)) __do_init(void) {
+ static _Bool __initialized;
+ if (__builtin_expect(__initialized, 0))
+ return;
+ __initialized = 1;
+
+#ifdef EH_USE_FRAME_REGISTRY
+ static struct { void *p[8]; } __object;
+ if (__register_frame_info)
+ __register_frame_info(__EH_FRAME_LIST__, &__object);
+#endif
+#ifndef CRT_HAS_INITFINI_ARRAY
+ const size_t n = __CTOR_LIST_END__ - __CTOR_LIST__ - 1;
+ for (size_t i = n; i >= 1; i--) __CTOR_LIST__[i]();
+#endif
+}
+
+#ifdef CRT_HAS_INITFINI_ARRAY
+__attribute__((section(".init_array"),
+ used)) static void (*__init)(void) = __do_init;
+#elif defined(__i386__) || defined(__x86_64__)
+__asm__(".pushsection .init,\"ax\",@progbits\n\t"
+ "call __do_init\n\t"
+ ".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .init,\"ax\",%progbits\n\t"
+ "call __do_init\n\t"
+ ".popsection");
+#elif defined(__arm__) || defined(__aarch64__)
+__asm__(".pushsection .init,\"ax\",%progbits\n\t"
+ "bl __do_init\n\t"
+ ".popsection");
+#elif defined(__mips__)
+__asm__(".pushsection .init,\"ax\",@progbits\n\t"
+ "jal __do_init\n\t"
+ ".popsection");
+#elif defined(__powerpc__) || defined(__powerpc64__)
+__asm__(".pushsection .init,\"ax\",@progbits\n\t"
+ "bl __do_init\n\t"
+ "nop\n\t"
+ ".popsection");
+#elif defined(__sparc__)
+__asm__(".pushsection .init,\"ax\",@progbits\n\t"
+ "call __do_init\n\t"
+ ".popsection");
+#else
+#error "crtbegin without .init_fini array unimplemented for this architecture"
+#endif // CRT_HAS_INITFINI_ARRAY
+
+#ifndef CRT_HAS_INITFINI_ARRAY
+static fp __DTOR_LIST__[]
+ __attribute__((section(".dtors"), aligned(sizeof(fp)))) = {(fp)-1};
+extern fp __DTOR_LIST_END__[];
+#endif
+
+static void __attribute__((used)) __do_fini(void) {
+ static _Bool __finalized;
+ if (__builtin_expect(__finalized, 0))
+ return;
+ __finalized = 1;
+
+ if (__cxa_finalize)
+ __cxa_finalize(__dso_handle);
+
+#ifndef CRT_HAS_INITFINI_ARRAY
+ const size_t n = __DTOR_LIST_END__ - __DTOR_LIST__ - 1;
+ for (size_t i = 1; i <= n; i++) __DTOR_LIST__[i]();
+#endif
+#ifdef EH_USE_FRAME_REGISTRY
+ if (__deregister_frame_info)
+ __deregister_frame_info(__EH_FRAME_LIST__);
+#endif
+}
+
+#ifdef CRT_HAS_INITFINI_ARRAY
+__attribute__((section(".fini_array"),
+ used)) static void (*__fini)(void) = __do_fini;
+#elif defined(__i386__) || defined(__x86_64__)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "call __do_fini\n\t"
+ ".popsection");
+#elif defined(__arm__) || defined(__aarch64__)
+__asm__(".pushsection .fini,\"ax\",%progbits\n\t"
+ "bl __do_fini\n\t"
+ ".popsection");
+#elif defined(__mips__)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "jal __do_fini\n\t"
+ ".popsection");
+#elif defined(__powerpc__) || defined(__powerpc64__)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "bl __do_fini\n\t"
+ "nop\n\t"
+ ".popsection");
+#elif defined(__riscv)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "call __do_fini\n\t"
+ ".popsection");
+#elif defined(__sparc__)
+__asm__(".pushsection .fini,\"ax\",@progbits\n\t"
+ "call __do_fini\n\t"
+ ".popsection");
+#else
+#error "crtbegin without .init_fini array unimplemented for this architecture"
+#endif // CRT_HAS_INIT_FINI_ARRAY
diff --git a/system/lib/compiler-rt/lib/builtins/crtend.c b/system/lib/compiler-rt/lib/builtins/crtend.c
new file mode 100644
index 0000000000000..ebcc60b89a102
--- /dev/null
+++ b/system/lib/compiler-rt/lib/builtins/crtend.c
@@ -0,0 +1,22 @@
+//===-- crtend.c - End of constructors and destructors --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include
+
+// Put 4-byte zero which is the length field in FDE at the end as a terminator.
+const int32_t __EH_FRAME_LIST_END__[]
+ __attribute__((section(".eh_frame"), aligned(sizeof(int32_t)),
+ visibility("hidden"), used)) = {0};
+
+#ifndef CRT_HAS_INITFINI_ARRAY
+typedef void (*fp)(void);
+fp __CTOR_LIST_END__[]
+ __attribute__((section(".ctors"), visibility("hidden"), used)) = {0};
+fp __DTOR_LIST_END__[]
+ __attribute__((section(".dtors"), visibility("hidden"), used)) = {0};
+#endif
diff --git a/system/lib/compiler-rt/lib/builtins/divtf3.c b/system/lib/compiler-rt/lib/builtins/divtf3.c
index 5bcc9a8e4aa18..bd76763b07d3a 100644
--- a/system/lib/compiler-rt/lib/builtins/divtf3.c
+++ b/system/lib/compiler-rt/lib/builtins/divtf3.c
@@ -14,7 +14,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#define NUMBER_OF_HALF_ITERATIONS 4
#define NUMBER_OF_FULL_ITERATIONS 1
diff --git a/system/lib/compiler-rt/lib/builtins/extenddftf2.c b/system/lib/compiler-rt/lib/builtins/extenddftf2.c
index ddf470ecd6293..835076be1f208 100644
--- a/system/lib/compiler-rt/lib/builtins/extenddftf2.c
+++ b/system/lib/compiler-rt/lib/builtins/extenddftf2.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#define SRC_DOUBLE
#define DST_QUAD
#include "fp_extend_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/extendhftf2.c b/system/lib/compiler-rt/lib/builtins/extendhftf2.c
index aefe9737d34f9..a2cb0f771ee9a 100644
--- a/system/lib/compiler-rt/lib/builtins/extendhftf2.c
+++ b/system/lib/compiler-rt/lib/builtins/extendhftf2.c
@@ -10,8 +10,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
- defined(COMPILER_RT_HAS_FLOAT16)
+#if defined(CRT_HAS_TF_MODE) && defined(COMPILER_RT_HAS_FLOAT16)
#define SRC_HALF
#define DST_QUAD
#include "fp_extend_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/extendsftf2.c b/system/lib/compiler-rt/lib/builtins/extendsftf2.c
index cf1fd2face20a..0739859bcbc18 100644
--- a/system/lib/compiler-rt/lib/builtins/extendsftf2.c
+++ b/system/lib/compiler-rt/lib/builtins/extendsftf2.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#define SRC_SINGLE
#define DST_QUAD
#include "fp_extend_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixtfdi.c b/system/lib/compiler-rt/lib/builtins/fixtfdi.c
index fe570e6b37555..d27a99b6f364e 100644
--- a/system/lib/compiler-rt/lib/builtins/fixtfdi.c
+++ b/system/lib/compiler-rt/lib/builtins/fixtfdi.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef di_int fixint_t;
typedef du_int fixuint_t;
#include "fp_fixint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixtfsi.c b/system/lib/compiler-rt/lib/builtins/fixtfsi.c
index a32bd964caa3b..01e352acc592d 100644
--- a/system/lib/compiler-rt/lib/builtins/fixtfsi.c
+++ b/system/lib/compiler-rt/lib/builtins/fixtfsi.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef si_int fixint_t;
typedef su_int fixuint_t;
#include "fp_fixint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixtfti.c b/system/lib/compiler-rt/lib/builtins/fixtfti.c
index 19f84ce389074..491fca502113d 100644
--- a/system/lib/compiler-rt/lib/builtins/fixtfti.c
+++ b/system/lib/compiler-rt/lib/builtins/fixtfti.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef ti_int fixint_t;
typedef tu_int fixuint_t;
#include "fp_fixint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfdi.c b/system/lib/compiler-rt/lib/builtins/fixunstfdi.c
index a0805e63db820..febdb8f5682fa 100644
--- a/system/lib/compiler-rt/lib/builtins/fixunstfdi.c
+++ b/system/lib/compiler-rt/lib/builtins/fixunstfdi.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef du_int fixuint_t;
#include "fp_fixuint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfsi.c b/system/lib/compiler-rt/lib/builtins/fixunstfsi.c
index 3a1320ed3e0a2..4efc387df453d 100644
--- a/system/lib/compiler-rt/lib/builtins/fixunstfsi.c
+++ b/system/lib/compiler-rt/lib/builtins/fixunstfsi.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef su_int fixuint_t;
#include "fp_fixuint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfti.c b/system/lib/compiler-rt/lib/builtins/fixunstfti.c
index 23cd1ab615a70..fa9e7aa07108b 100644
--- a/system/lib/compiler-rt/lib/builtins/fixunstfti.c
+++ b/system/lib/compiler-rt/lib/builtins/fixunstfti.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
typedef tu_int fixuint_t;
#include "fp_fixuint_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/floatdidf.c b/system/lib/compiler-rt/lib/builtins/floatdidf.c
index d37c43b1f2f92..c994aad3f079e 100644
--- a/system/lib/compiler-rt/lib/builtins/floatdidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatdidf.c
@@ -50,7 +50,7 @@ COMPILER_RT_ABI double __floatdidf(di_int a) {
return 0.0;
const unsigned N = sizeof(di_int) * CHAR_BIT;
const di_int s = a >> (N - 1);
- a = (a ^ s) - s;
+ a = (du_int)(a ^ s) - s;
int sd = N - __builtin_clzll(a); // number of significant digits
int e = sd - 1; // exponent
if (sd > DBL_MANT_DIG) {
diff --git a/system/lib/compiler-rt/lib/builtins/floatdisf.c b/system/lib/compiler-rt/lib/builtins/floatdisf.c
index 5c6316431e394..0b62ed8689bc6 100644
--- a/system/lib/compiler-rt/lib/builtins/floatdisf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatdisf.c
@@ -24,7 +24,7 @@ COMPILER_RT_ABI float __floatdisf(di_int a) {
return 0.0F;
const unsigned N = sizeof(di_int) * CHAR_BIT;
const di_int s = a >> (N - 1);
- a = (a ^ s) - s;
+ a = (du_int)(a ^ s) - s;
int sd = N - __builtin_clzll(a); // number of significant digits
si_int e = sd - 1; // exponent
if (sd > FLT_MANT_DIG) {
diff --git a/system/lib/compiler-rt/lib/builtins/floatditf.c b/system/lib/compiler-rt/lib/builtins/floatditf.c
index 9b07b65825b88..c6e326a1923a6 100644
--- a/system/lib/compiler-rt/lib/builtins/floatditf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatditf.c
@@ -15,7 +15,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floatditf(di_int a) {
const int aWidth = sizeof a * CHAR_BIT;
diff --git a/system/lib/compiler-rt/lib/builtins/floatsitf.c b/system/lib/compiler-rt/lib/builtins/floatsitf.c
index 80a4ef08fb0ed..4d5b52f4ed919 100644
--- a/system/lib/compiler-rt/lib/builtins/floatsitf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatsitf.c
@@ -15,7 +15,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floatsitf(si_int a) {
const int aWidth = sizeof a * CHAR_BIT;
diff --git a/system/lib/compiler-rt/lib/builtins/floattidf.c b/system/lib/compiler-rt/lib/builtins/floattidf.c
index 0a1c04bec82ea..7bfe87f53aa02 100644
--- a/system/lib/compiler-rt/lib/builtins/floattidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floattidf.c
@@ -29,7 +29,7 @@ COMPILER_RT_ABI double __floattidf(ti_int a) {
const ti_int s = a >> (N - 1);
a = (a ^ s) - s;
int sd = N - __clzti2(a); // number of significant digits
- int e = sd - 1; // exponent
+ si_int e = sd - 1; // exponent
if (sd > DBL_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floattisf.c b/system/lib/compiler-rt/lib/builtins/floattisf.c
index a8fcdbe14c073..717cb361f075a 100644
--- a/system/lib/compiler-rt/lib/builtins/floattisf.c
+++ b/system/lib/compiler-rt/lib/builtins/floattisf.c
@@ -28,7 +28,7 @@ COMPILER_RT_ABI float __floattisf(ti_int a) {
const ti_int s = a >> (N - 1);
a = (a ^ s) - s;
int sd = N - __clzti2(a); // number of significant digits
- int e = sd - 1; // exponent
+ si_int e = sd - 1; // exponent
if (sd > FLT_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floattitf.c b/system/lib/compiler-rt/lib/builtins/floattitf.c
index 196cbdae14e02..fff0755c3bb46 100644
--- a/system/lib/compiler-rt/lib/builtins/floattitf.c
+++ b/system/lib/compiler-rt/lib/builtins/floattitf.c
@@ -25,7 +25,7 @@
// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm
// mmmm mmmm mmmm
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floattitf(ti_int a) {
if (a == 0)
return 0.0;
@@ -34,7 +34,7 @@ COMPILER_RT_ABI fp_t __floattitf(ti_int a) {
a = (a ^ s) - s;
int sd = N - __clzti2(a); // number of significant digits
int e = sd - 1; // exponent
- if (sd > LDBL_MANT_DIG) {
+ if (sd > TF_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
// 12345678901234567890123456
@@ -43,27 +43,27 @@ COMPILER_RT_ABI fp_t __floattitf(ti_int a) {
// Q = bit LDBL_MANT_DIG bits to the right of 1
// R = "or" of all bits to the right of Q
switch (sd) {
- case LDBL_MANT_DIG + 1:
+ case TF_MANT_DIG + 1:
a <<= 1;
break;
- case LDBL_MANT_DIG + 2:
+ case TF_MANT_DIG + 2:
break;
default:
- a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) |
- ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0);
+ a = ((tu_int)a >> (sd - (TF_MANT_DIG + 2))) |
+ ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0);
};
// finish:
a |= (a & 4) != 0; // Or P into R
++a; // round - this step may add a significant bit
a >>= 2; // dump Q and R
// a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits
- if (a & ((tu_int)1 << LDBL_MANT_DIG)) {
+ if (a & ((tu_int)1 << TF_MANT_DIG)) {
a >>= 1;
++e;
}
// a is now rounded to LDBL_MANT_DIG bits
} else {
- a <<= (LDBL_MANT_DIG - sd);
+ a <<= (TF_MANT_DIG - sd);
// a is now rounded to LDBL_MANT_DIG bits
}
diff --git a/system/lib/compiler-rt/lib/builtins/floatunditf.c b/system/lib/compiler-rt/lib/builtins/floatunditf.c
index 8d310851e179c..abe0ca9ed8c50 100644
--- a/system/lib/compiler-rt/lib/builtins/floatunditf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatunditf.c
@@ -15,7 +15,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floatunditf(du_int a) {
const int aWidth = sizeof a * CHAR_BIT;
diff --git a/system/lib/compiler-rt/lib/builtins/floatunsitf.c b/system/lib/compiler-rt/lib/builtins/floatunsitf.c
index 7ba1fb6000dcc..3f0a5249fddd3 100644
--- a/system/lib/compiler-rt/lib/builtins/floatunsitf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatunsitf.c
@@ -15,7 +15,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floatunsitf(su_int a) {
const int aWidth = sizeof a * CHAR_BIT;
diff --git a/system/lib/compiler-rt/lib/builtins/floatuntidf.c b/system/lib/compiler-rt/lib/builtins/floatuntidf.c
index e69e65c1ace4e..4dfca8e493098 100644
--- a/system/lib/compiler-rt/lib/builtins/floatuntidf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatuntidf.c
@@ -27,7 +27,7 @@ COMPILER_RT_ABI double __floatuntidf(tu_int a) {
return 0.0;
const unsigned N = sizeof(tu_int) * CHAR_BIT;
int sd = N - __clzti2(a); // number of significant digits
- int e = sd - 1; // exponent
+ si_int e = sd - 1; // exponent
if (sd > DBL_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floatuntisf.c b/system/lib/compiler-rt/lib/builtins/floatuntisf.c
index 9dec0ab5c58fc..a53659cd1fcac 100644
--- a/system/lib/compiler-rt/lib/builtins/floatuntisf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatuntisf.c
@@ -26,7 +26,7 @@ COMPILER_RT_ABI float __floatuntisf(tu_int a) {
return 0.0F;
const unsigned N = sizeof(tu_int) * CHAR_BIT;
int sd = N - __clzti2(a); // number of significant digits
- int e = sd - 1; // exponent
+ si_int e = sd - 1; // exponent
if (sd > FLT_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
diff --git a/system/lib/compiler-rt/lib/builtins/floatuntitf.c b/system/lib/compiler-rt/lib/builtins/floatuntitf.c
index d308d3118d03d..33a81b34eeb19 100644
--- a/system/lib/compiler-rt/lib/builtins/floatuntitf.c
+++ b/system/lib/compiler-rt/lib/builtins/floatuntitf.c
@@ -25,44 +25,44 @@
// mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm
// mmmm mmmm mmmm
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) {
if (a == 0)
return 0.0;
const unsigned N = sizeof(tu_int) * CHAR_BIT;
int sd = N - __clzti2(a); // number of significant digits
int e = sd - 1; // exponent
- if (sd > LDBL_MANT_DIG) {
+ if (sd > TF_MANT_DIG) {
// start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx
// finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR
// 12345678901234567890123456
// 1 = msb 1 bit
- // P = bit LDBL_MANT_DIG-1 bits to the right of 1
- // Q = bit LDBL_MANT_DIG bits to the right of 1
+ // P = bit TF_MANT_DIG-1 bits to the right of 1
+ // Q = bit TF_MANT_DIG bits to the right of 1
// R = "or" of all bits to the right of Q
switch (sd) {
- case LDBL_MANT_DIG + 1:
+ case TF_MANT_DIG + 1:
a <<= 1;
break;
- case LDBL_MANT_DIG + 2:
+ case TF_MANT_DIG + 2:
break;
default:
- a = (a >> (sd - (LDBL_MANT_DIG + 2))) |
- ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0);
+ a = (a >> (sd - (TF_MANT_DIG + 2))) |
+ ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0);
};
// finish:
a |= (a & 4) != 0; // Or P into R
++a; // round - this step may add a significant bit
a >>= 2; // dump Q and R
- // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits
- if (a & ((tu_int)1 << LDBL_MANT_DIG)) {
+ // a is now rounded to TF_MANT_DIG or TF_MANT_DIG+1 bits
+ if (a & ((tu_int)1 << TF_MANT_DIG)) {
a >>= 1;
++e;
}
- // a is now rounded to LDBL_MANT_DIG bits
+ // a is now rounded to TF_MANT_DIG bits
} else {
- a <<= (LDBL_MANT_DIG - sd);
- // a is now rounded to LDBL_MANT_DIG bits
+ a <<= (TF_MANT_DIG - sd);
+ // a is now rounded to TF_MANT_DIG bits
}
long_double_bits fb;
diff --git a/system/lib/compiler-rt/lib/builtins/fp_lib.h b/system/lib/compiler-rt/lib/builtins/fp_lib.h
index 3fb13a033a14c..58eb45fcc7292 100644
--- a/system/lib/compiler-rt/lib/builtins/fp_lib.h
+++ b/system/lib/compiler-rt/lib/builtins/fp_lib.h
@@ -106,7 +106,13 @@ COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b);
#elif defined QUAD_PRECISION
#if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__)
+// TODO: Availability of the *tf functions should not depend on long double
+// being IEEE 128, but instead on being able to use a 128-bit floating-point
+// type, which includes __float128.
+// Right now this (incorrectly) stops the builtins from being used for x86.
#define CRT_LDBL_128BIT
+#define CRT_HAS_TF_MODE
+#define TF_C(c) c##L
typedef uint64_t half_rep_t;
typedef __uint128_t rep_t;
typedef __int128_t srep_t;
@@ -116,6 +122,7 @@ typedef long double fp_t;
// Note: Since there is no explicit way to tell compiler the constant is a
// 128-bit integer, we let the constant be casted to 128-bit integer
#define significandBits 112
+#define TF_MANT_DIG (significandBits + 1)
static __inline int rep_clz(rep_t a) {
const union {
diff --git a/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c b/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c
index 58fd7ceb58c35..ef63a5fb83472 100644
--- a/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c
+++ b/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c
@@ -219,7 +219,7 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0(
}
// Walk call-site table looking for range that includes current PC.
uint8_t callSiteEncoding = *lsda++;
- uint32_t callSiteTableLength = readULEB128(&lsda);
+ size_t callSiteTableLength = readULEB128(&lsda);
const uint8_t *callSiteTableStart = lsda;
const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength;
const uint8_t *p = callSiteTableStart;
diff --git a/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc
index 567d8b9e6e603..27e7c8c43d600 100644
--- a/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc
+++ b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc
@@ -18,10 +18,10 @@
static __inline fixint_t __muloXi4(fixint_t a, fixint_t b, int *overflow) {
const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
- const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1));
const fixint_t MAX = ~MIN;
*overflow = 0;
- fixint_t result = a * b;
+ fixint_t result = (fixuint_t)a * b;
if (a == MIN) {
if (b != 0 && b != 1)
*overflow = 1;
diff --git a/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc
index 1e920716ec499..06559cf302ea6 100644
--- a/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc
+++ b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc
@@ -18,7 +18,7 @@
static __inline fixint_t __mulvXi3(fixint_t a, fixint_t b) {
const int N = (int)(sizeof(fixint_t) * CHAR_BIT);
- const fixint_t MIN = (fixint_t)1 << (N - 1);
+ const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1));
const fixint_t MAX = ~MIN;
if (a == MIN) {
if (b == 0 || b == 1)
diff --git a/system/lib/compiler-rt/lib/builtins/lshrti3.c b/system/lib/compiler-rt/lib/builtins/lshrti3.c
index d00a220959939..5dc8a0a2347f2 100644
--- a/system/lib/compiler-rt/lib/builtins/lshrti3.c
+++ b/system/lib/compiler-rt/lib/builtins/lshrti3.c
@@ -18,7 +18,7 @@
// Precondition: 0 <= b < bits_in_tword
-COMPILER_RT_ABI ti_int __lshrti3(ti_int a, si_int b) {
+COMPILER_RT_ABI ti_int __lshrti3(ti_int a, int b) {
const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT);
utwords input;
utwords result;
diff --git a/system/lib/compiler-rt/lib/builtins/mulodi4.c b/system/lib/compiler-rt/lib/builtins/mulodi4.c
index 7209676a327e4..6ecf92664fb5c 100644
--- a/system/lib/compiler-rt/lib/builtins/mulodi4.c
+++ b/system/lib/compiler-rt/lib/builtins/mulodi4.c
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#define fixint_t di_int
+#define fixuint_t du_int
#include "int_mulo_impl.inc"
// Returns: a * b
diff --git a/system/lib/compiler-rt/lib/builtins/mulosi4.c b/system/lib/compiler-rt/lib/builtins/mulosi4.c
index 4e03c24455d67..3fd18a122a463 100644
--- a/system/lib/compiler-rt/lib/builtins/mulosi4.c
+++ b/system/lib/compiler-rt/lib/builtins/mulosi4.c
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#define fixint_t si_int
+#define fixuint_t su_int
#include "int_mulo_impl.inc"
// Returns: a * b
diff --git a/system/lib/compiler-rt/lib/builtins/muloti4.c b/system/lib/compiler-rt/lib/builtins/muloti4.c
index 9a7aa85b022bf..9aab6fc3efb33 100644
--- a/system/lib/compiler-rt/lib/builtins/muloti4.c
+++ b/system/lib/compiler-rt/lib/builtins/muloti4.c
@@ -19,6 +19,7 @@
// Effects: sets *overflow to 1 if a * b overflows
#define fixint_t ti_int
+#define fixuint_t tu_int
#include "int_mulo_impl.inc"
COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) {
diff --git a/system/lib/compiler-rt/lib/builtins/multf3.c b/system/lib/compiler-rt/lib/builtins/multf3.c
index 0626fb8c7fc9c..8fd73688712cb 100644
--- a/system/lib/compiler-rt/lib/builtins/multf3.c
+++ b/system/lib/compiler-rt/lib/builtins/multf3.c
@@ -14,7 +14,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#include "fp_mul_impl.inc"
COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { return __mulXf3__(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/mulvdi3.c b/system/lib/compiler-rt/lib/builtins/mulvdi3.c
index 1d672c6dc155d..d787d297d564e 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvdi3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvdi3.c
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#define fixint_t di_int
+#define fixuint_t du_int
#include "int_mulv_impl.inc"
// Returns: a * b
diff --git a/system/lib/compiler-rt/lib/builtins/mulvsi3.c b/system/lib/compiler-rt/lib/builtins/mulvsi3.c
index 00b2e50eeca91..2571881195fcc 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvsi3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvsi3.c
@@ -11,6 +11,7 @@
//===----------------------------------------------------------------------===//
#define fixint_t si_int
+#define fixuint_t su_int
#include "int_mulv_impl.inc"
// Returns: a * b
diff --git a/system/lib/compiler-rt/lib/builtins/mulvti3.c b/system/lib/compiler-rt/lib/builtins/mulvti3.c
index ba355149f9a76..fad9b2ae2765b 100644
--- a/system/lib/compiler-rt/lib/builtins/mulvti3.c
+++ b/system/lib/compiler-rt/lib/builtins/mulvti3.c
@@ -19,6 +19,7 @@
// Effects: aborts if a * b overflows
#define fixint_t ti_int
+#define fixuint_t tu_int
#include "int_mulv_impl.inc"
COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { return __mulvXi3(a, b); }
diff --git a/system/lib/compiler-rt/lib/builtins/negvdi2.c b/system/lib/compiler-rt/lib/builtins/negvdi2.c
index 5c52b3ec2aa60..8c1cf2fa58d46 100644
--- a/system/lib/compiler-rt/lib/builtins/negvdi2.c
+++ b/system/lib/compiler-rt/lib/builtins/negvdi2.c
@@ -17,7 +17,8 @@
// Effects: aborts if -a overflows
COMPILER_RT_ABI di_int __negvdi2(di_int a) {
- const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1);
+ const di_int MIN =
+ (di_int)((du_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1));
if (a == MIN)
compilerrt_abort();
return -a;
diff --git a/system/lib/compiler-rt/lib/builtins/negvsi2.c b/system/lib/compiler-rt/lib/builtins/negvsi2.c
index cccdee6dc5e55..70f214f9761d8 100644
--- a/system/lib/compiler-rt/lib/builtins/negvsi2.c
+++ b/system/lib/compiler-rt/lib/builtins/negvsi2.c
@@ -17,7 +17,8 @@
// Effects: aborts if -a overflows
COMPILER_RT_ABI si_int __negvsi2(si_int a) {
- const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1);
+ const si_int MIN =
+ (si_int)((su_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1));
if (a == MIN)
compilerrt_abort();
return -a;
diff --git a/system/lib/compiler-rt/lib/builtins/os_version_check.c b/system/lib/compiler-rt/lib/builtins/os_version_check.c
index ebfb2dfc72ddd..182eabe7a6ae2 100644
--- a/system/lib/compiler-rt/lib/builtins/os_version_check.c
+++ b/system/lib/compiler-rt/lib/builtins/os_version_check.c
@@ -86,6 +86,10 @@ typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex,
CFStringEncoding);
typedef void (*CFReleaseFuncTy)(CFTypeRef);
+extern __attribute__((weak_import))
+bool _availability_version_check(uint32_t count,
+ dyld_build_version_t versions[]);
+
static void _initializeAvailabilityCheck(bool LoadPlist) {
if (AvailabilityVersionCheck && !LoadPlist) {
// New API is supported and we're not being asked to load the plist,
@@ -94,8 +98,8 @@ static void _initializeAvailabilityCheck(bool LoadPlist) {
}
// Use the new API if it's is available.
- AvailabilityVersionCheck = (AvailabilityVersionCheckFuncTy)dlsym(
- RTLD_DEFAULT, "_availability_version_check");
+ if (_availability_version_check)
+ AvailabilityVersionCheck = &_availability_version_check;
if (AvailabilityVersionCheck && !LoadPlist) {
// New API is supported and we're not being asked to load the plist,
diff --git a/system/lib/compiler-rt/lib/builtins/powitf2.c b/system/lib/compiler-rt/lib/builtins/powitf2.c
index 8e639a03a3c4b..74fe707a4e8c4 100644
--- a/system/lib/compiler-rt/lib/builtins/powitf2.c
+++ b/system/lib/compiler-rt/lib/builtins/powitf2.c
@@ -13,7 +13,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
// Returns: a ^ b
diff --git a/system/lib/compiler-rt/lib/builtins/subtf3.c b/system/lib/compiler-rt/lib/builtins/subtf3.c
index 3364c28f81792..e1b1022034bf7 100644
--- a/system/lib/compiler-rt/lib/builtins/subtf3.c
+++ b/system/lib/compiler-rt/lib/builtins/subtf3.c
@@ -13,7 +13,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b);
// Subtraction; flip the sign bit of b and add.
diff --git a/system/lib/compiler-rt/lib/builtins/trunctfdf2.c b/system/lib/compiler-rt/lib/builtins/trunctfdf2.c
index 6857ea54d8a57..f0d2e4141f3b0 100644
--- a/system/lib/compiler-rt/lib/builtins/trunctfdf2.c
+++ b/system/lib/compiler-rt/lib/builtins/trunctfdf2.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#define SRC_QUAD
#define DST_DOUBLE
#include "fp_trunc_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/trunctfhf2.c b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c
index e3a2309d954bc..f7776327251c7 100644
--- a/system/lib/compiler-rt/lib/builtins/trunctfhf2.c
+++ b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c
@@ -10,8 +10,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \
- defined(COMPILER_RT_HAS_FLOAT16)
+#if defined(CRT_HAS_TF_MODE) && defined(COMPILER_RT_HAS_FLOAT16)
#define SRC_QUAD
#define DST_HALF
#include "fp_trunc_impl.inc"
diff --git a/system/lib/compiler-rt/lib/builtins/trunctfsf2.c b/system/lib/compiler-rt/lib/builtins/trunctfsf2.c
index 0261b1e90f5da..242735f738c13 100644
--- a/system/lib/compiler-rt/lib/builtins/trunctfsf2.c
+++ b/system/lib/compiler-rt/lib/builtins/trunctfsf2.c
@@ -9,7 +9,7 @@
#define QUAD_PRECISION
#include "fp_lib.h"
-#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT)
+#if defined(CRT_HAS_TF_MODE)
#define SRC_QUAD
#define DST_SINGLE
#include "fp_trunc_impl.inc"
diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h
index 370e4ab844a6b..0bb06c88b9451 100644
--- a/system/lib/compiler-rt/lib/interception/interception.h
+++ b/system/lib/compiler-rt/lib/interception/interception.h
@@ -14,9 +14,10 @@
#ifndef INTERCEPTION_H
#define INTERCEPTION_H
+#include "sanitizer_common/sanitizer_asm.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
-#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \
+#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \
!SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \
!SANITIZER_SOLARIS && !SANITIZER_EMSCRIPTEN
# error "Interception doesn't work on this operating system."
@@ -67,24 +68,50 @@ typedef __sanitizer::OFF64_T OFF64_T;
// for more details). To intercept such functions you need to use the
// INTERCEPTOR_WITH_SUFFIX(...) macro.
-// How it works:
-// To replace system functions on Linux we just need to declare functions
-// with same names in our library and then obtain the real function pointers
+// How it works on Linux
+// ---------------------
+//
+// To replace system functions on Linux we just need to declare functions with
+// the same names in our library and then obtain the real function pointers
// using dlsym().
-// There is one complication. A user may also intercept some of the functions
-// we intercept. To resolve this we declare our interceptors with __interceptor_
-// prefix, and then make actual interceptors weak aliases to __interceptor_
-// functions.
//
-// This is not so on Mac OS, where the two-level namespace makes
-// our replacement functions invisible to other libraries. This may be overcomed
-// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared
-// libraries in Chromium were noticed when doing so.
+// There is one complication: a user may also intercept some of the functions we
+// intercept. To allow for up to 3 interceptors (including ours) of a given
+// function "func", the interceptor implementation is in ___interceptor_func,
+// which is aliased by a weak function __interceptor_func, which in turn is
+// aliased (via a trampoline) by weak wrapper function "func".
+//
+// Most user interceptors should define a foreign interceptor as follows:
+//
+// - provide a non-weak function "func" that performs interception;
+// - if __interceptor_func exists, call it to perform the real functionality;
+// - if it does not exist, figure out the real function and call it instead.
+//
+// In rare cases, a foreign interceptor (of another dynamic analysis runtime)
+// may be defined as follows (on supported architectures):
+//
+// - provide a non-weak function __interceptor_func that performs interception;
+// - if ___interceptor_func exists, call it to perform the real functionality;
+// - if it does not exist, figure out the real function and call it instead;
+// - provide a weak function "func" that is an alias to __interceptor_func.
+//
+// With this protocol, sanitizer interceptors, foreign user interceptors, and
+// foreign interceptors of other dynamic analysis runtimes, or any combination
+// thereof, may co-exist simultaneously.
+//
+// How it works on Mac OS
+// ----------------------
+//
+// This is not so on Mac OS, where the two-level namespace makes our replacement
+// functions invisible to other libraries. This may be overcomed using the
+// DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared libraries in
+// Chromium were noticed when doing so.
+//
// Instead we create a dylib containing a __DATA,__interpose section that
// associates library functions with their wrappers. When this dylib is
-// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all
-// the calls to interposed functions done through stubs to the wrapper
-// functions.
+// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all the
+// calls to interposed functions done through stubs to the wrapper functions.
+//
// As it's decided at compile time which functions are to be intercepted on Mac,
// INTERCEPT_FUNCTION() is effectively a no-op on this system.
@@ -100,58 +127,106 @@ struct interpose_substitution {
// For a function foo() create a global pair of pointers { wrap_foo, foo } in
// the __DATA,__interpose section.
// As a result all the calls to foo() will be routed to wrap_foo() at runtime.
-#define INTERPOSER(func_name) __attribute__((used)) \
+#define INTERPOSER(func_name) __attribute__((used)) \
const interpose_substitution substitution_##func_name[] \
__attribute__((section("__DATA, __interpose"))) = { \
- { reinterpret_cast(WRAP(func_name)), \
- reinterpret_cast(func_name) } \
+ { reinterpret_cast(WRAP(func_name)), \
+ reinterpret_cast(func_name) } \
}
// For a function foo() and a wrapper function bar() create a global pair
// of pointers { bar, foo } in the __DATA,__interpose section.
// As a result all the calls to foo() will be routed to bar() at runtime.
#define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \
-const interpose_substitution substitution_##func_name[] \
- __attribute__((section("__DATA, __interpose"))) = { \
- { reinterpret_cast(wrapper_name), \
- reinterpret_cast(func_name) } \
+const interpose_substitution substitution_##func_name[] \
+ __attribute__((section("__DATA, __interpose"))) = { \
+ { reinterpret_cast(wrapper_name), \
+ reinterpret_cast(func_name) } \
}
# define WRAP(x) wrap_##x
-# define WRAPPER_NAME(x) "wrap_"#x
+# define TRAMPOLINE(x) WRAP(x)
# define INTERCEPTOR_ATTRIBUTE
# define DECLARE_WRAPPER(ret_type, func, ...)
#elif SANITIZER_WINDOWS
# define WRAP(x) __asan_wrap_##x
-# define WRAPPER_NAME(x) "__asan_wrap_"#x
+# define TRAMPOLINE(x) WRAP(x)
# define INTERCEPTOR_ATTRIBUTE __declspec(dllexport)
-# define DECLARE_WRAPPER(ret_type, func, ...) \
+# define DECLARE_WRAPPER(ret_type, func, ...) \
extern "C" ret_type func(__VA_ARGS__);
-# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
+# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \
extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__);
#elif SANITIZER_EMSCRIPTEN
# define WRAP(x) x
-# define WRAPPER_NAME(x) #x
# define INTERCEPTOR_ATTRIBUTE
# define DECLARE_WRAPPER(ret_type, func, ...)
-#elif SANITIZER_FREEBSD || SANITIZER_NETBSD
-# define WRAP(x) __interceptor_ ## x
-# define WRAPPER_NAME(x) "__interceptor_" #x
+#elif !SANITIZER_FUCHSIA // LINUX, FREEBSD, NETBSD, SOLARIS
# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
+# if ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
+// Weak aliases of weak aliases do not work, therefore we need to set up a
+// trampoline function. The function "func" is a weak alias to the trampoline
+// (so that we may check if "func" was overridden), which calls the weak
+// function __interceptor_func, which in turn aliases the actual interceptor
+// implementation ___interceptor_func:
+//
+// [wrapper "func": weak] --(alias)--> [TRAMPOLINE(func)]
+// |
+// +--------(tail call)-------+
+// |
+// v
+// [__interceptor_func: weak] --(alias)--> [WRAP(func)]
+//
+// We use inline assembly to define most of this, because not all compilers
+// support functions with the "naked" attribute with every architecture.
+# define WRAP(x) ___interceptor_ ## x
+# define TRAMPOLINE(x) __interceptor_trampoline_ ## x
+# if SANITIZER_FREEBSD || SANITIZER_NETBSD
// FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher
// priority than weak ones so weak aliases won't work for indirect calls
// in position-independent (-fPIC / -fPIE) mode.
-# define DECLARE_WRAPPER(ret_type, func, ...) \
- extern "C" ret_type func(__VA_ARGS__) \
- __attribute__((alias("__interceptor_" #func), visibility("default")));
-#elif !SANITIZER_FUCHSIA
-# define WRAP(x) __interceptor_ ## x
-# define WRAPPER_NAME(x) "__interceptor_" #x
-# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default")))
-# define DECLARE_WRAPPER(ret_type, func, ...) \
- extern "C" ret_type func(__VA_ARGS__) \
- __attribute__((weak, alias("__interceptor_" #func), visibility("default")));
+# define __ASM_WEAK_WRAPPER(func) ".globl " #func "\n"
+# else
+# define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n"
+# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD
+// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h
+# define DECLARE_WRAPPER(ret_type, func, ...) \
+ extern "C" ret_type func(__VA_ARGS__); \
+ extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
+ extern "C" ret_type __interceptor_##func(__VA_ARGS__) \
+ INTERCEPTOR_ATTRIBUTE __attribute__((weak)) ALIAS(WRAP(func)); \
+ asm( \
+ ".text\n" \
+ __ASM_WEAK_WRAPPER(func) \
+ ".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
+ ".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
+ ".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n" \
+ SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \
+ SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \
+ SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \
+ SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \
+ SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \
+ ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \
+ ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \
+ );
+# else // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
+// Some architectures cannot implement efficient interceptor trampolines with
+// just a plain jump due to complexities of resolving a preemptible symbol. In
+// those cases, revert to just this scheme:
+//
+// [wrapper "func": weak] --(alias)--> [WRAP(func)]
+//
+# define WRAP(x) __interceptor_ ## x
+# define TRAMPOLINE(x) WRAP(x)
+# if SANITIZER_FREEBSD || SANITIZER_NETBSD
+# define __ATTRIBUTE_WEAK_WRAPPER
+# else
+# define __ATTRIBUTE_WEAK_WRAPPER __attribute__((weak))
+# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD
+# define DECLARE_WRAPPER(ret_type, func, ...) \
+ extern "C" ret_type func(__VA_ARGS__) \
+ INTERCEPTOR_ATTRIBUTE __ATTRIBUTE_WEAK_WRAPPER ALIAS(WRAP(func));
+# endif // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT
#endif
#if SANITIZER_FUCHSIA
@@ -174,10 +249,10 @@ const interpose_substitution substitution_##func_name[] \
# define REAL(x) __interception::PTR_TO_REAL(x)
# define FUNC_TYPE(x) x##_type
-# define DECLARE_REAL(ret_type, func, ...) \
+# define DECLARE_REAL(ret_type, func, ...) \
typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \
- namespace __interception { \
- extern FUNC_TYPE(func) PTR_TO_REAL(func); \
+ namespace __interception { \
+ extern FUNC_TYPE(func) PTR_TO_REAL(func); \
}
# define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src)
#else // SANITIZER_APPLE
@@ -188,14 +263,16 @@ const interpose_substitution substitution_##func_name[] \
#endif // SANITIZER_APPLE
#if !SANITIZER_FUCHSIA
-# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
+# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \
DECLARE_REAL(ret_type, func, __VA_ARGS__) \
+ extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
extern "C" ret_type WRAP(func)(__VA_ARGS__);
// Declare an interceptor and its wrapper defined in a different translation
// unit (ex. asm).
-# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
- extern "C" ret_type WRAP(func)(__VA_ARGS__); \
- extern "C" ret_type func(__VA_ARGS__);
+# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \
+ extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \
+ extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+ extern "C" ret_type func(__VA_ARGS__);
#else
# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...)
# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...)
@@ -227,12 +304,10 @@ const interpose_substitution substitution_##func_name[] \
#elif !SANITIZER_APPLE
-#define INTERCEPTOR(ret_type, func, ...) \
- DEFINE_REAL(ret_type, func, __VA_ARGS__) \
- DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
- extern "C" \
- INTERCEPTOR_ATTRIBUTE \
- ret_type WRAP(func)(__VA_ARGS__)
+#define INTERCEPTOR(ret_type, func, ...) \
+ DEFINE_REAL(ret_type, func, __VA_ARGS__) \
+ DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \
+ extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
// We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now.
#define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \
@@ -240,10 +315,10 @@ const interpose_substitution substitution_##func_name[] \
#else // SANITIZER_APPLE
-#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
- extern "C" ret_type func(__VA_ARGS__) suffix; \
- extern "C" ret_type WRAP(func)(__VA_ARGS__); \
- INTERPOSER(func); \
+#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \
+ extern "C" ret_type func(__VA_ARGS__) suffix; \
+ extern "C" ret_type WRAP(func)(__VA_ARGS__); \
+ INTERPOSER(func); \
extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__)
#define INTERCEPTOR(ret_type, func, ...) \
@@ -258,14 +333,12 @@ const interpose_substitution substitution_##func_name[] \
#endif
#if SANITIZER_WINDOWS
-# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
+# define INTERCEPTOR_WINAPI(ret_type, func, ...) \
typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \
- namespace __interception { \
- FUNC_TYPE(func) PTR_TO_REAL(func); \
- } \
- extern "C" \
- INTERCEPTOR_ATTRIBUTE \
- ret_type __stdcall WRAP(func)(__VA_ARGS__)
+ namespace __interception { \
+ FUNC_TYPE(func) PTR_TO_REAL(func); \
+ } \
+ extern "C" INTERCEPTOR_ATTRIBUTE ret_type __stdcall WRAP(func)(__VA_ARGS__)
#endif
// ISO C++ forbids casting between pointer-to-function and pointer-to-object,
diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.cpp b/system/lib/compiler-rt/lib/interception/interception_linux.cpp
index 5111a87f0a6c9..ef8136eb4fc70 100644
--- a/system/lib/compiler-rt/lib/interception/interception_linux.cpp
+++ b/system/lib/compiler-rt/lib/interception/interception_linux.cpp
@@ -33,7 +33,7 @@ static int StrCmp(const char *s1, const char *s2) {
}
#endif
-static void *GetFuncAddr(const char *name, uptr wrapper_addr) {
+static void *GetFuncAddr(const char *name, uptr trampoline) {
#if SANITIZER_NETBSD
// FIXME: Find a better way to handle renames
if (StrCmp(name, "sigaction"))
@@ -50,17 +50,17 @@ static void *GetFuncAddr(const char *name, uptr wrapper_addr) {
// In case `name' is not loaded, dlsym ends up finding the actual wrapper.
// We don't want to intercept the wrapper and have it point to itself.
- if ((uptr)addr == wrapper_addr)
+ if ((uptr)addr == trampoline)
addr = nullptr;
}
return addr;
}
bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func,
- uptr wrapper) {
- void *addr = GetFuncAddr(name, wrapper);
+ uptr trampoline) {
+ void *addr = GetFuncAddr(name, trampoline);
*ptr_to_real = (uptr)addr;
- return addr && (func == wrapper);
+ return addr && (func == trampoline);
}
// dlvsym is a GNU extension supported by some other platforms.
@@ -70,12 +70,12 @@ static void *GetFuncAddr(const char *name, const char *ver) {
}
bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
- uptr func, uptr wrapper) {
+ uptr func, uptr trampoline) {
void *addr = GetFuncAddr(name, ver);
*ptr_to_real = (uptr)addr;
- return addr && (func == wrapper);
+ return addr && (func == trampoline);
}
-#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
+# endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
} // namespace __interception
diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.h b/system/lib/compiler-rt/lib/interception/interception_linux.h
index 6d3957e570267..caa42fc4b7ccd 100644
--- a/system/lib/compiler-rt/lib/interception/interception_linux.h
+++ b/system/lib/compiler-rt/lib/interception/interception_linux.h
@@ -15,7 +15,7 @@
SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN
#if !defined(INCLUDED_FROM_INTERCEPTION_LIB)
-# error "interception_linux.h should be included from interception library only"
+# error interception_linux.h should be included from interception library only
#endif
#ifndef INTERCEPTION_LINUX_H
@@ -23,26 +23,26 @@
namespace __interception {
bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func,
- uptr wrapper);
+ uptr trampoline);
bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real,
- uptr func, uptr wrapper);
+ uptr func, uptr trampoline);
} // namespace __interception
#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \
::__interception::InterceptFunction( \
#func, \
- (::__interception::uptr *) & REAL(func), \
- (::__interception::uptr) & (func), \
- (::__interception::uptr) & WRAP(func))
+ (::__interception::uptr *)&REAL(func), \
+ (::__interception::uptr)&(func), \
+ (::__interception::uptr)&TRAMPOLINE(func))
// dlvsym is a GNU extension supported by some other platforms.
#if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
::__interception::InterceptFunction( \
#func, symver, \
- (::__interception::uptr *) & REAL(func), \
- (::__interception::uptr) & (func), \
- (::__interception::uptr) & WRAP(func))
+ (::__interception::uptr *)&REAL(func), \
+ (::__interception::uptr)&(func), \
+ (::__interception::uptr)&TRAMPOLINE(func))
#else
#define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \
INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func)
diff --git a/system/lib/compiler-rt/lib/interception/interception_win.cpp b/system/lib/compiler-rt/lib/interception/interception_win.cpp
index faaa8ee15381d..00c317510e420 100644
--- a/system/lib/compiler-rt/lib/interception/interception_win.cpp
+++ b/system/lib/compiler-rt/lib/interception/interception_win.cpp
@@ -141,8 +141,29 @@ static const int kBranchLength =
FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength);
static const int kDirectBranchLength = kBranchLength + kAddressLength;
+# if defined(_MSC_VER)
+# define INTERCEPTION_FORMAT(f, a)
+# else
+# define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a)))
+# endif
+
+static void (*ErrorReportCallback)(const char *format, ...)
+ INTERCEPTION_FORMAT(1, 2);
+
+void SetErrorReportCallback(void (*callback)(const char *format, ...)) {
+ ErrorReportCallback = callback;
+}
+
+# define ReportError(...) \
+ do { \
+ if (ErrorReportCallback) \
+ ErrorReportCallback(__VA_ARGS__); \
+ } while (0)
+
static void InterceptionFailed() {
- // Do we have a good way to abort with an error message here?
+ ReportError("interception_win: failed due to an unrecoverable error.\n");
+ // This acts like an abort when no debugger is attached. According to an old
+ // comment, calling abort() leads to an infinite recursion in CheckFailed.
__debugbreak();
}
@@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) {
}
static void WriteJumpInstruction(uptr from, uptr target) {
- if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target))
+ if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) {
+ ReportError(
+ "interception_win: cannot write jmp further than 2GB away, from %p to "
+ "%p.\n",
+ (void *)from, (void *)target);
InterceptionFailed();
+ }
ptrdiff_t offset = target - from - kJumpInstructionLength;
*(u8*)from = 0xE9;
*(u32*)(from + 1) = offset;
@@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) {
int offset = indirect_target - from - kIndirectJumpInstructionLength;
if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength,
indirect_target)) {
+ ReportError(
+ "interception_win: cannot write indirect jmp with target further than "
+ "2GB away, from %p to %p.\n",
+ (void *)from, (void *)indirect_target);
InterceptionFailed();
}
*(u16*)from = 0x25FF;
@@ -492,6 +522,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
case 0xFF8B: // 8B FF : mov edi, edi
case 0xEC8B: // 8B EC : mov ebp, esp
case 0xc889: // 89 C8 : mov eax, ecx
+ case 0xE589: // 89 E5 : mov ebp, esp
case 0xC18B: // 8B C1 : mov eax, ecx
case 0xC033: // 33 C0 : xor eax, eax
case 0xC933: // 33 C9 : xor ecx, ecx
@@ -641,6 +672,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX]
case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX]
case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX]
+ case 0x245C8B: // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX]
+ case 0x246C8B: // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX]
case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX]
case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX]
return 4;
@@ -652,12 +685,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) {
}
#endif
- // Unknown instruction!
- // FIXME: Unknown instruction failures might happen when we add a new
- // interceptor or a new compiler version. In either case, they should result
- // in visible and readable error messages. However, merely calling abort()
- // leads to an infinite recursion in CheckFailed.
- InterceptionFailed();
+ // Unknown instruction! This might happen when we add a new interceptor, use
+ // a new compiler version, or if Windows changed how some functions are
+ // compiled. In either case, we print the address and 8 bytes of instructions
+ // to notify the user about the error and to help identify the unknown
+ // instruction. Don't treat this as a fatal error, though we can break the
+ // debugger if one has been attached.
+ u8 *bytes = (u8 *)address;
+ ReportError(
+ "interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x "
+ "%02x %02x %02x\n",
+ (void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4],
+ bytes[5], bytes[6], bytes[7]);
+ if (::IsDebuggerPresent())
+ __debugbreak();
return 0;
}
@@ -678,6 +719,8 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) {
while (cursor != size) {
size_t rel_offset = 0;
size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset);
+ if (!instruction_size)
+ return false;
_memcpy((void*)(to + cursor), (void*)(from + cursor),
(size_t)instruction_size);
if (rel_offset) {
@@ -895,6 +938,10 @@ static void **InterestingDLLsAvailable() {
"msvcr120.dll", // VS2013
"vcruntime140.dll", // VS2015
"ucrtbase.dll", // Universal CRT
+#if (defined(__MINGW32__) && defined(__i386__))
+ "libc++.dll", // libc++
+ "libunwind.dll", // libunwind
+#endif
// NTDLL should go last as it exports some functions that we should
// override in the CRT [presumably only used internally].
"ntdll.dll", NULL};
diff --git a/system/lib/compiler-rt/lib/interception/interception_win.h b/system/lib/compiler-rt/lib/interception/interception_win.h
index 4590013019e37..f6eca82191cba 100644
--- a/system/lib/compiler-rt/lib/interception/interception_win.h
+++ b/system/lib/compiler-rt/lib/interception/interception_win.h
@@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch,
const char *function_name, uptr new_function,
uptr *orig_old_func);
+// Sets a callback to be used for reporting errors by interception_win. The
+// callback will be called with printf-like arguments. Intended to be used with
+// __sanitizer::Report. Pass nullptr to disable error reporting (default).
+void SetErrorReportCallback(void (*callback)(const char *format, ...));
+
#if !SANITIZER_WINDOWS64
// Exposed for unittests
bool OverrideFunctionWithDetour(
diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cpp b/system/lib/compiler-rt/lib/lsan/lsan.cpp
index 8ec353d071382..6a20a128e021e 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan.cpp
@@ -41,7 +41,7 @@ void __sanitizer::BufferedStackTrace::UnwindImpl(
uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) {
using namespace __lsan;
uptr stack_top = 0, stack_bottom = 0;
- if (ThreadContext *t = CurrentThreadContext()) {
+ if (ThreadContextLsanBase *t = GetCurrentThread()) {
stack_top = t->stack_end();
stack_bottom = t->stack_begin();
}
@@ -117,7 +117,7 @@ extern "C" void __lsan_init() {
ReplaceSystemMalloc();
InitTlsSize();
InitializeInterceptors();
- InitializeThreadRegistry();
+ InitializeThreads();
#if !SANITIZER_EMSCRIPTEN
// Emscripten does not have signals
InstallDeadlySignalHandlers(LsanOnDeadlySignal);
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
index 9ab77a640b7ff..2b5ef0480facf 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp
@@ -49,8 +49,11 @@ void InitializeAllocator() {
max_malloc_size = kMaxAllowedMallocSize;
}
+void AllocatorThreadStart() { allocator.InitCache(GetAllocatorCache()); }
+
void AllocatorThreadFinish() {
allocator.SwallowCache(GetAllocatorCache());
+ allocator.DestroyCache(GetAllocatorCache());
}
static ChunkMetadata *Metadata(const void *p) {
@@ -65,12 +68,14 @@ static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) {
m->stack_trace_id = StackDepotPut(stack);
m->requested_size = size;
atomic_store(reinterpret_cast(m), 1, memory_order_relaxed);
+ RunMallocHooks(p, size);
}
static void RegisterDeallocation(void *p) {
if (!p) return;
ChunkMetadata *m = Metadata(p);
CHECK(m);
+ RunFreeHooks(p);
atomic_store(reinterpret_cast(m), 0, memory_order_relaxed);
}
@@ -104,7 +109,6 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
if (cleared && allocator.FromPrimary(p))
memset(p, 0, size);
RegisterAllocation(stack, p, size);
- RunMallocHooks(p, size);
return p;
}
@@ -119,7 +123,6 @@ static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
}
void Deallocate(void *p) {
- RunFreeHooks(p);
RegisterDeallocation(p);
allocator.Deallocate(GetAllocatorCache(), p);
}
@@ -145,6 +148,22 @@ void GetAllocatorCacheRange(uptr *begin, uptr *end) {
*end = *begin + sizeof(AllocatorCache);
}
+static const void *GetMallocBegin(const void *p) {
+ if (!p)
+ return nullptr;
+ void *beg = allocator.GetBlockBegin(p);
+ if (!beg)
+ return nullptr;
+ ChunkMetadata *m = Metadata(beg);
+ if (!m)
+ return nullptr;
+ if (!m->allocated)
+ return nullptr;
+ if (m->requested_size == 0)
+ return nullptr;
+ return (const void *)beg;
+}
+
uptr GetMallocUsableSize(const void *p) {
if (!p)
return 0;
@@ -153,6 +172,10 @@ uptr GetMallocUsableSize(const void *p) {
return m->requested_size;
}
+uptr GetMallocUsableSizeFast(const void *p) {
+ return Metadata(p)->requested_size;
+}
+
int lsan_posix_memalign(void **memptr, uptr alignment, uptr size,
const StackTrace &stack) {
if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
@@ -275,6 +298,10 @@ uptr GetUserBegin(uptr chunk) {
return chunk;
}
+uptr GetUserAddr(uptr chunk) {
+ return chunk;
+}
+
LsanMetadata::LsanMetadata(uptr chunk) {
metadata_ = Metadata(reinterpret_cast(chunk));
CHECK(metadata_);
@@ -304,7 +331,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) {
allocator.ForEachChunk(callback, arg);
}
-IgnoreObjectResult IgnoreObjectLocked(const void *p) {
+IgnoreObjectResult IgnoreObject(const void *p) {
void *chunk = allocator.GetBlockBegin(p);
if (!chunk || p < chunk) return kIgnoreObjectInvalid;
ChunkMetadata *m = Metadata(chunk);
@@ -319,15 +346,6 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) {
}
}
-void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) {
- // This function can be used to treat memory reachable from `tctx` as live.
- // This is useful for threads that have been created but not yet started.
-
- // This is currently a no-op because the LSan `pthread_create()` interceptor
- // blocks until the child thread starts which keeps the thread's `arg` pointer
- // live.
-}
-
} // namespace __lsan
using namespace __lsan;
@@ -348,7 +366,7 @@ uptr __sanitizer_get_heap_size() {
}
SANITIZER_INTERFACE_ATTRIBUTE
-uptr __sanitizer_get_free_bytes() { return 0; }
+uptr __sanitizer_get_free_bytes() { return 1; }
SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_get_unmapped_bytes() { return 0; }
@@ -357,11 +375,29 @@ SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
SANITIZER_INTERFACE_ATTRIBUTE
-int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; }
+int __sanitizer_get_ownership(const void *p) {
+ return GetMallocBegin(p) != nullptr;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+const void * __sanitizer_get_allocated_begin(const void *p) {
+ return GetMallocBegin(p);
+}
SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_get_allocated_size(const void *p) {
return GetMallocUsableSize(p);
}
+SANITIZER_INTERFACE_ATTRIBUTE
+uptr __sanitizer_get_allocated_size_fast(const void *p) {
+ DCHECK_EQ(p, __sanitizer_get_allocated_begin(p));
+ uptr ret = GetMallocUsableSizeFast(p);
+ DCHECK_EQ(ret, __sanitizer_get_allocated_size(p));
+ return ret;
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
+
} // extern "C"
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
index b67d9d7750efc..84cce4c6baebf 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h
@@ -32,6 +32,7 @@ template
void ForEachChunk(const Callable &callback);
void GetAllocatorCacheRange(uptr *begin, uptr *end);
+void AllocatorThreadStart();
void AllocatorThreadFinish();
void InitializeAllocator();
@@ -68,13 +69,13 @@ using PrimaryAllocator = PrimaryAllocatorASVT;
# if SANITIZER_FUCHSIA || defined(__powerpc64__)
const uptr kAllocatorSpace = ~(uptr)0;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
-#elif defined(__s390x__)
-const uptr kAllocatorSpace = 0x40000000000ULL;
-const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
-# else
+# elif SANITIZER_APPLE
const uptr kAllocatorSpace = 0x600000000000ULL;
const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
-# endif
+# else
+const uptr kAllocatorSpace = 0x500000000000ULL;
+const uptr kAllocatorSize = 0x40000000000ULL; // 4T.
+# endif
template
struct AP64 { // Allocator64 parameters. Deliberately using a short name.
static const uptr kSpaceBeg = kAllocatorSpace;
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
index 8d6f3b9f73a4b..d3dfec5ede1b5 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp
@@ -39,15 +39,13 @@
# else
# define OBJC_DATA_MASK 0x00007ffffffffff8UL
# endif
-// https://github.com/apple-oss-distributions/objc4/blob/8701d5672d3fd3cd817aeb84db1077aafe1a1604/runtime/objc-runtime-new.h#L139
-# define OBJC_FAST_IS_RW 0x8000000000000000UL
# endif
namespace __lsan {
// This mutex is used to prevent races between DoLeakCheck and IgnoreObject, and
// also to protect the global list of root regions.
-Mutex global_mutex;
+static Mutex global_mutex;
Flags lsan_flags;
@@ -178,13 +176,11 @@ static uptr GetCallerPC(const StackTrace &stack) {
}
# if SANITIZER_APPLE
-// Objective-C class data pointers are stored with flags in the low bits, so
-// they need to be transformed back into something that looks like a pointer.
-static inline void *MaybeTransformPointer(void *p) {
+// Several pointers in the Objective-C runtime (method cache and class_rw_t,
+// for example) are tagged with additional bits we need to strip.
+static inline void *TransformPointer(void *p) {
uptr ptr = reinterpret_cast(p);
- if ((ptr & OBJC_FAST_IS_RW) == OBJC_FAST_IS_RW)
- ptr &= OBJC_DATA_MASK;
- return reinterpret_cast(ptr);
+ return reinterpret_cast(ptr & OBJC_DATA_MASK);
}
# endif
@@ -254,12 +250,6 @@ static LeakSuppressionContext *GetSuppressionContext() {
return suppression_ctx;
}
-static InternalMmapVectorNoCtor root_regions;
-
-InternalMmapVectorNoCtor const *GetRootRegions() {
- return &root_regions;
-}
-
void InitCommonLsan() {
if (common_flags()->detect_leaks) {
// Initialization which can fail or print warnings should only be done if
@@ -283,13 +273,22 @@ static inline bool MaybeUserPointer(uptr p) {
if (p < kMinAddress)
return false;
# if defined(__x86_64__)
- // Accept only canonical form user-space addresses.
- return ((p >> 47) == 0);
+ // TODO: support LAM48 and 5 level page tables.
+ // LAM_U57 mask format
+ // * top byte: 0x81 because the format is: [0] [6-bit tag] [0]
+ // * top-1 byte: 0xff because it should be 0
+ // * top-2 byte: 0x80 because Linux uses 128 TB VMA ending at 0x7fffffffffff
+ constexpr uptr kLAM_U57Mask = 0x81ff80;
+ constexpr uptr kPointerMask = kLAM_U57Mask << 40;
+ return ((p & kPointerMask) == 0);
# elif defined(__mips64)
return ((p >> 40) == 0);
# elif defined(__aarch64__)
+ // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in
+ // address translation and can be used to store a tag.
+ constexpr uptr kPointerMask = 255ULL << 48;
// Accept up to 48 bit VMA.
- return ((p >> 48) == 0);
+ return ((p & kPointerMask) == 0);
# elif defined(__loongarch_lp64)
// Allow 47-bit user-space VMA at current.
return ((p >> 47) == 0);
@@ -325,7 +324,7 @@ void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier,
for (; pp + sizeof(void *) <= end; pp += alignment) { // NOLINT
void *p = *reinterpret_cast(pp);
# if SANITIZER_APPLE
- p = MaybeTransformPointer(p);
+ p = TransformPointer(p);
# endif
if (!MaybeUserPointer(reinterpret_cast(p)))
continue;
@@ -557,42 +556,56 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
# endif // SANITIZER_FUCHSIA
-void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
- uptr region_begin, uptr region_end, bool is_readable) {
- uptr intersection_begin = Max(root_region.begin, region_begin);
- uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
- if (intersection_begin >= intersection_end)
- return;
- LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
- (void *)root_region.begin,
- (void *)(root_region.begin + root_region.size),
- (void *)region_begin, (void *)region_end,
- is_readable ? "readable" : "unreadable");
- if (is_readable)
- ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT",
- kReachable);
+// A map that contains [region_begin, region_end) pairs.
+using RootRegions = DenseMap, uptr>;
+
+static RootRegions &GetRootRegionsLocked() {
+ global_mutex.CheckLocked();
+ static RootRegions *regions = nullptr;
+ alignas(RootRegions) static char placeholder[sizeof(RootRegions)];
+ if (!regions)
+ regions = new (placeholder) RootRegions();
+ return *regions;
}
-static void ProcessRootRegion(Frontier *frontier,
- const RootRegion &root_region) {
-#if SANITIZER_EMSCRIPTEN
- ScanRootRegion(frontier, root_region, 0, emscripten_get_heap_size(), true);
-#else
- MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
- MemoryMappedSegment segment;
- while (proc_maps.Next(&segment)) {
- ScanRootRegion(frontier, root_region, segment.start, segment.end,
- segment.IsReadable());
+bool HasRootRegions() { return !GetRootRegionsLocked().empty(); }
+
+void ScanRootRegions(Frontier *frontier,
+ const InternalMmapVectorNoCtor &mapped_regions) {
+ if (!flags()->use_root_regions)
+ return;
+
+ InternalMmapVector regions;
+ GetRootRegionsLocked().forEach([&](const auto &kv) {
+ regions.push_back({kv.first.first, kv.first.second});
+ return true;
+ });
+
+ InternalMmapVector intersection;
+ Intersect(mapped_regions, regions, intersection);
+
+ for (const Region &r : intersection) {
+ LOG_POINTERS("Root region intersects with mapped region at %p-%p\n",
+ (void *)r.begin, (void *)r.end);
+ ScanRangeForPointers(r.begin, r.end, frontier, "ROOT", kReachable);
}
-#endif // SANITIZER_EMSCRIPTEN
}
// Scans root regions for heap pointers.
static void ProcessRootRegions(Frontier *frontier) {
- if (!flags()->use_root_regions)
+ if (!flags()->use_root_regions || !HasRootRegions())
return;
- for (uptr i = 0; i < root_regions.size(); i++)
- ProcessRootRegion(frontier, root_regions[i]);
+ InternalMmapVector mapped_regions;
+#if SANITIZER_EMSCRIPTEN
+ mapped_regions.push_back({0, emscripten_get_heap_size()});
+#else
+ MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
+ MemoryMappedSegment segment;
+ while (proc_maps.Next(&segment))
+ if (segment.IsReadable())
+ mapped_regions.push_back({segment.start, segment.end});
+#endif // SANITIZER_EMSCRIPTEN
+ ScanRootRegions(frontier, mapped_regions);
}
static void FloodFillTag(Frontier *frontier, ChunkTag tag) {
@@ -897,7 +910,7 @@ void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) {
leaks_.push_back(leak);
}
if (flags()->report_objects) {
- LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
+ LeakedObject obj = {leaks_[i].id, GetUserAddr(chunk), leaked_size};
leaked_objects_.push_back(obj);
}
}
@@ -982,7 +995,7 @@ void LeakReport::PrintSummary() {
uptr LeakReport::ApplySuppressions() {
LeakSuppressionContext *suppressions = GetSuppressionContext();
- uptr new_suppressions = false;
+ uptr new_suppressions = 0;
for (uptr i = 0; i < leaks_.size(); i++) {
if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count,
leaks_[i].total_size)) {
@@ -1031,7 +1044,7 @@ void __lsan_ignore_object(const void *p) {
// Cannot use PointsIntoChunk or LsanMetadata here, since the allocator is not
// locked.
Lock l(&global_mutex);
- IgnoreObjectResult res = IgnoreObjectLocked(p);
+ IgnoreObjectResult res = IgnoreObject(p);
if (res == kIgnoreObjectInvalid)
VReport(1, "__lsan_ignore_object(): no heap object found at %p\n", p);
if (res == kIgnoreObjectAlreadyIgnored)
@@ -1047,36 +1060,37 @@ void __lsan_ignore_object(const void *p) {
SANITIZER_INTERFACE_ATTRIBUTE
void __lsan_register_root_region(const void *begin, uptr size) {
#if CAN_SANITIZE_LEAKS
- Lock l(&global_mutex);
- RootRegion region = {reinterpret_cast(begin), size};
- root_regions.push_back(region);
VReport(1, "Registered root region at %p of size %zu\n", begin, size);
+ uptr b = reinterpret_cast(begin);
+ uptr e = b + size;
+ CHECK_LT(b, e);
+
+ Lock l(&global_mutex);
+ ++GetRootRegionsLocked()[{b, e}];
#endif // CAN_SANITIZE_LEAKS
}
SANITIZER_INTERFACE_ATTRIBUTE
void __lsan_unregister_root_region(const void *begin, uptr size) {
#if CAN_SANITIZE_LEAKS
- Lock l(&global_mutex);
- bool removed = false;
- for (uptr i = 0; i < root_regions.size(); i++) {
- RootRegion region = root_regions[i];
- if (region.begin == reinterpret_cast(begin) && region.size == size) {
- removed = true;
- uptr last_index = root_regions.size() - 1;
- root_regions[i] = root_regions[last_index];
- root_regions.pop_back();
- VReport(1, "Unregistered root region at %p of size %zu\n", begin, size);
- break;
+ uptr b = reinterpret_cast(begin);
+ uptr e = b + size;
+ CHECK_LT(b, e);
+ VReport(1, "Unregistered root region at %p of size %zu\n", begin, size);
+
+ {
+ Lock l(&global_mutex);
+ if (auto *f = GetRootRegionsLocked().find({b, e})) {
+ if (--(f->second) == 0)
+ GetRootRegionsLocked().erase(f);
+ return;
}
}
- if (!removed) {
- Report(
- "__lsan_unregister_root_region(): region at %p of size %zu has not "
- "been registered.\n",
- begin, size);
- Die();
- }
+ Report(
+ "__lsan_unregister_root_region(): region at %p of size %zu has not "
+ "been registered.\n",
+ begin, size);
+ Die();
#endif // CAN_SANITIZE_LEAKS
}
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.h b/system/lib/compiler-rt/lib/lsan/lsan_common.h
index 86eb7d84a8891..46b7e6f34025c 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common.h
@@ -18,6 +18,7 @@
#include "sanitizer_common/sanitizer_common.h"
#include "sanitizer_common/sanitizer_internal_defs.h"
#include "sanitizer_common/sanitizer_platform.h"
+#include "sanitizer_common/sanitizer_range.h"
#include "sanitizer_common/sanitizer_stackdepot.h"
#include "sanitizer_common/sanitizer_stoptheworld.h"
#include "sanitizer_common/sanitizer_symbolizer.h"
@@ -79,11 +80,6 @@ enum IgnoreObjectResult {
kIgnoreObjectInvalid
};
-struct Range {
- uptr begin;
- uptr end;
-};
-
//// --------------------------------------------------------------------------
//// Poisoning prototypes.
//// --------------------------------------------------------------------------
@@ -96,8 +92,8 @@ bool WordIsPoisoned(uptr addr);
//// --------------------------------------------------------------------------
// Wrappers for ThreadRegistry access.
-void LockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
-void UnlockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+void LockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
+void UnlockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS;
// If called from the main thread, updates the main thread's TID in the thread
// registry. We need this to handle processes that fork() without a subsequent
// exec(), which invalidates the recorded TID. To update it, we must call
@@ -131,6 +127,9 @@ void GetAllocatorGlobalRange(uptr *begin, uptr *end);
uptr PointsIntoChunk(void *p);
// Returns address of user-visible chunk contained in this allocator chunk.
uptr GetUserBegin(uptr chunk);
+// Returns user-visible address for chunk. If memory tagging is used this
+// function will return the tagged address.
+uptr GetUserAddr(uptr chunk);
// Wrapper for chunk metadata operations.
class LsanMetadata {
@@ -151,19 +150,19 @@ class LsanMetadata {
void ForEachChunk(ForEachChunkCallback callback, void *arg);
// Helper for __lsan_ignore_object().
-IgnoreObjectResult IgnoreObjectLocked(const void *p);
+IgnoreObjectResult IgnoreObject(const void *p);
// The rest of the LSan interface which is implemented by library.
struct ScopedStopTheWorldLock {
ScopedStopTheWorldLock() {
- LockThreadRegistry();
+ LockThreads();
LockAllocator();
}
~ScopedStopTheWorldLock() {
UnlockAllocator();
- UnlockThreadRegistry();
+ UnlockThreads();
}
ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete;
@@ -236,11 +235,6 @@ void InitializePlatformSpecificModules();
void ProcessGlobalRegions(Frontier *frontier);
void ProcessPlatformSpecificAllocations(Frontier *frontier);
-struct RootRegion {
- uptr begin;
- uptr size;
-};
-
// LockStuffAndStopTheWorld can start to use Scan* calls to collect into
// this Frontier vector before the StopTheWorldCallback actually runs.
// This is used when the OS has a unified callback API for suspending
@@ -253,9 +247,11 @@ struct CheckForLeaksParam {
bool success = false;
};
-InternalMmapVectorNoCtor const *GetRootRegions();
-void ScanRootRegion(Frontier *frontier, RootRegion const ®ion,
- uptr region_begin, uptr region_end, bool is_readable);
+using Region = Range;
+
+bool HasRootRegions();
+void ScanRootRegions(Frontier *frontier,
+ const InternalMmapVectorNoCtor ®ion);
// Run stoptheworld while holding any platform-specific locks, as well as the
// allocator and thread registry locks.
void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
index f2b1301577238..2407842cb5828 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp
@@ -105,14 +105,14 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback,
CheckForLeaksParam *argument) {
// Currently, on Emscripten this does nothing and just calls the callback.
// This works fine on a single-threaded environment.
- LockThreadRegistry();
+ LockThreads();
LockAllocator();
StopTheWorld(callback, argument);
UnlockAllocator();
- UnlockThreadRegistry();
+ UnlockThreads();
}
-u32 GetCurrentThread();
+u32 GetCurrentThreadId();
// This is based on ProcessThreads in lsan_common.cc.
// We changed this to be a callback that gets called per thread by
@@ -142,7 +142,7 @@ static void ProcessThreadsCallback(ThreadContextBase *tctx, void *arg) {
// We can't get the SP for other threads to narrow down the range, but we
// we can for the current thread.
- if (tctx->tid == GetCurrentThread()) {
+ if (tctx->tid == GetCurrentThreadId()) {
uptr sp = (uptr) __builtin_frame_address(0);
if (sp < stack_begin || sp >= stack_end) {
// SP is outside the recorded stack range (e.g. the thread is running a
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
index b6b15095744d7..4e5198979b954 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp
@@ -25,6 +25,8 @@
# include "sanitizer_common/sanitizer_allocator_internal.h"
namespace __lsan {
+class ThreadContextLsanBase;
+
enum class SeenRegion {
None = 0,
AllocOnce = 1 << 0,
@@ -50,18 +52,18 @@ struct RegionScanState {
typedef struct {
int disable_counter;
- u32 current_thread_id;
+ ThreadContextLsanBase *current_thread;
AllocatorCache cache;
} thread_local_data_t;
static pthread_key_t key;
static pthread_once_t key_once = PTHREAD_ONCE_INIT;
-// The main thread destructor requires the current thread id,
-// so we can't destroy it until it's been used and reset to invalid tid
+// The main thread destructor requires the current thread,
+// so we can't destroy it until it's been used and reset.
void restore_tid_data(void *ptr) {
thread_local_data_t *data = (thread_local_data_t *)ptr;
- if (data->current_thread_id != kInvalidTid)
+ if (data->current_thread)
pthread_setspecific(key, data);
}
@@ -76,7 +78,7 @@ static thread_local_data_t *get_tls_val(bool alloc) {
if (ptr == NULL && alloc) {
ptr = (thread_local_data_t *)InternalAlloc(sizeof(*ptr));
ptr->disable_counter = 0;
- ptr->current_thread_id = kInvalidTid;
+ ptr->current_thread = nullptr;
ptr->cache = AllocatorCache();
pthread_setspecific(key, ptr);
}
@@ -99,12 +101,14 @@ void EnableInThisThread() {
--*disable_counter;
}
-u32 GetCurrentThread() {
+ThreadContextLsanBase *GetCurrentThread() {
thread_local_data_t *data = get_tls_val(false);
- return data ? data->current_thread_id : kInvalidTid;
+ return data ? data->current_thread : nullptr;
}
-void SetCurrentThread(u32 tid) { get_tls_val(true)->current_thread_id = tid; }
+void SetCurrentThread(ThreadContextLsanBase *tctx) {
+ get_tls_val(true)->current_thread = tctx;
+}
AllocatorCache *GetAllocatorCache() { return &get_tls_val(true)->cache; }
@@ -161,7 +165,8 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
vm_address_t address = 0;
kern_return_t err = KERN_SUCCESS;
- InternalMmapVectorNoCtor const *root_regions = GetRootRegions();
+ InternalMmapVector mapped_regions;
+ bool use_root_regions = flags()->use_root_regions && HasRootRegions();
RegionScanState scan_state;
while (err == KERN_SUCCESS) {
@@ -199,8 +204,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
// Recursing over the full memory map is very slow, break out
// early if we don't need the full iteration.
- if (scan_state.seen_regions == SeenRegion::All &&
- !(flags()->use_root_regions && root_regions->size() > 0)) {
+ if (scan_state.seen_regions == SeenRegion::All && !use_root_regions) {
break;
}
@@ -211,15 +215,12 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
//
// TODO(fjricci) - remove this once sanitizer_procmaps_mac has the same
// behavior as sanitizer_procmaps_linux and traverses all memory regions
- if (flags()->use_root_regions) {
- for (uptr i = 0; i < root_regions->size(); i++) {
- ScanRootRegion(frontier, (*root_regions)[i], address, end_address,
- info.protection & kProtectionRead);
- }
- }
+ if (use_root_regions && (info.protection & kProtectionRead))
+ mapped_regions.push_back({address, end_address});
address = end_address;
}
+ ScanRootRegions(frontier, mapped_regions);
}
// On darwin, we can intercept _exit gracefully, and return a failing exit code
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp
index 03ac0afbabff7..4edac9757a9c4 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp
@@ -46,6 +46,7 @@ struct OnStartedArgs {
};
void ThreadContext::OnStarted(void *arg) {
+ ThreadContextLsanBase::OnStarted(arg);
auto args = reinterpret_cast(arg);
cache_begin_ = args->cache_begin;
cache_end_ = args->cache_end;
@@ -98,7 +99,7 @@ void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached,
OnCreatedArgs args;
args.stack_begin = reinterpret_cast(stack_base);
args.stack_end = args.stack_begin + stack_size;
- u32 parent_tid = GetCurrentThread();
+ u32 parent_tid = GetCurrentThreadId();
u32 tid = ThreadCreate(parent_tid, detached, &args);
return reinterpret_cast(static_cast(tid));
}
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
index 996cf50f8f094..216fb420d80f4 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -40,6 +40,7 @@ int emscripten_builtin_pthread_create(void *thread, void *attr,
void *(*callback)(void *), void *arg);
int emscripten_builtin_pthread_join(void *th, void **ret);
int emscripten_builtin_pthread_detach(void *th);
+void emscripten_builtin_pthread_exit(void *th);
}
#endif
@@ -208,7 +209,7 @@ INTERCEPTOR(void*, pvalloc, uptr size) {
#endif // SANITIZER_INTERCEPT_PVALLOC
#if SANITIZER_INTERCEPT_CFREE
-INTERCEPTOR(void, cfree, void *p) ALIAS(WRAPPER_NAME(free));
+INTERCEPTOR(void, cfree, void *p) ALIAS(WRAP(free));
#define LSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree)
#else
#define LSAN_MAYBE_INTERCEPT_CFREE
@@ -426,16 +427,10 @@ INTERCEPTOR(char *, strerror, int errnum) {
#if SANITIZER_POSIX
-struct ThreadParam {
- void *(*callback)(void *arg);
- void *param;
- atomic_uintptr_t tid;
-};
-
-extern "C" void *__lsan_thread_start_func(void *arg) {
- ThreadParam *p = (ThreadParam*)arg;
- void* (*callback)(void *arg) = p->callback;
- void *param = p->param;
+template
+static void *ThreadStartFunc(void *arg) {
+ u32 parent_tid = (uptr)arg;
+ uptr tid = ThreadCreate(parent_tid, Detached);
// Wait until the last iteration to maximize the chance that we are the last
// destructor to run.
#if !SANITIZER_NETBSD && !SANITIZER_FREEBSD
@@ -444,74 +439,118 @@ extern "C" void *__lsan_thread_start_func(void *arg) {
Report("LeakSanitizer: failed to set thread key.\n");
Die();
}
-#endif
- int tid = 0;
- while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
- internal_sched_yield();
+# endif
ThreadStart(tid, GetTid());
-#if SANITIZER_EMSCRIPTEN
- emscripten_builtin_free(p);
-#else
- atomic_store(&p->tid, 0, memory_order_release);
-#endif
- return callback(param);
+ auto self = GetThreadSelf();
+ auto args = GetThreadArgRetval().GetArgs(self);
+ void *retval = (*args.routine)(args.arg_retval);
+ GetThreadArgRetval().Finish(self, retval);
+ return retval;
}
INTERCEPTOR(int, pthread_create, void *th, void *attr,
void *(*callback)(void *), void *param) {
ENSURE_LSAN_INITED;
EnsureMainThreadIDIsCorrect();
+
+#if !SANITIZER_EMSCRIPTEN
+ bool detached = [attr]() {
+ int d = 0;
+ return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d);
+ }();
+#endif
+
__sanitizer_pthread_attr_t myattr;
+#if SANITIZER_EMSCRIPTEN
if (!attr || attr == __ATTRP_C11_THREAD) {
+#else
+ if (!attr) {
+#endif
pthread_attr_init(&myattr);
attr = &myattr;
}
AdjustStackSize(attr);
- int detached = 0;
- pthread_attr_getdetachstate(attr, &detached);
#if SANITIZER_EMSCRIPTEN
- ThreadParam *p = (ThreadParam *) emscripten_builtin_malloc(sizeof(ThreadParam));
- p->callback = callback;
- p->param = param;
- atomic_store(&p->tid, 0, memory_order_relaxed);
-#else
- ThreadParam p;
- p.callback = callback;
- p.param = param;
- atomic_store(&p.tid, 0, memory_order_relaxed);
+ // In Emscripten sanitizer, attr can be nonzero but __ATTRP_C11_THREAD, in
+ // which case pthread_attr_getdetachstate needs to run after pthread_attr_init
+ // above.
+ bool detached = [attr]() {
+ int d = 0;
+ return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d);
+ }();
#endif
- int res;
+ uptr this_tid = GetCurrentThreadId();
+ int result;
{
// Ignore all allocations made by pthread_create: thread stack/TLS may be
// stored by pthread for future reuse even after thread destruction, and
// the linked list it's stored in doesn't even hold valid pointers to the
// objects, the latter are calculated by obscure pointer arithmetic.
ScopedInterceptorDisabler disabler;
-#if SANITIZER_EMSCRIPTEN
- res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, p);
-#else
- res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, &p);
-#endif
- }
- if (res == 0) {
- int tid = ThreadCreate(GetCurrentThread(), IsStateDetached(detached));
- CHECK_NE(tid, kMainTid);
-#if SANITIZER_EMSCRIPTEN
- atomic_store(&p->tid, tid, memory_order_release);
-#else
- atomic_store(&p.tid, tid, memory_order_release);
- while (atomic_load(&p.tid, memory_order_acquire) != 0)
- internal_sched_yield();
-#endif
+ GetThreadArgRetval().Create(detached, {callback, param}, [&]() -> uptr {
+ result = REAL(pthread_create)(
+ th, attr, detached ? ThreadStartFunc : ThreadStartFunc,
+ (void *)this_tid);
+ return result ? 0 : *(uptr *)(th);
+ });
}
if (attr == &myattr)
pthread_attr_destroy(&myattr);
- return res;
-}
-
-INTERCEPTOR(int, pthread_join, void *t, void **arg) {
- return REAL(pthread_join)(t, arg);
-}
+ return result;
+}
+
+INTERCEPTOR(int, pthread_join, void *thread, void **retval) {
+ int result;
+ GetThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_join)(thread, retval);
+ return !result;
+ });
+ return result;
+}
+
+INTERCEPTOR(int, pthread_detach, void *thread) {
+ int result;
+ GetThreadArgRetval().Detach((uptr)thread, [&]() {
+ result = REAL(pthread_detach)(thread);
+ return !result;
+ });
+ return result;
+}
+
+INTERCEPTOR(void, pthread_exit, void *retval) {
+ GetThreadArgRetval().Finish(GetThreadSelf(), retval);
+ REAL(pthread_exit)(retval);
+}
+
+# if SANITIZER_INTERCEPT_TRYJOIN
+INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) {
+ int result;
+ GetThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_tryjoin_np)(thread, ret);
+ return !result;
+ });
+ return result;
+}
+# define LSAN_MAYBE_INTERCEPT_TRYJOIN INTERCEPT_FUNCTION(pthread_tryjoin_np)
+# else
+# define LSAN_MAYBE_INTERCEPT_TRYJOIN
+# endif // SANITIZER_INTERCEPT_TRYJOIN
+
+# if SANITIZER_INTERCEPT_TIMEDJOIN
+INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret,
+ const struct timespec *abstime) {
+ int result;
+ GetThreadArgRetval().Join((uptr)thread, [&]() {
+ result = REAL(pthread_timedjoin_np)(thread, ret, abstime);
+ return !result;
+ });
+ return result;
+}
+# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN \
+ INTERCEPT_FUNCTION(pthread_timedjoin_np)
+# else
+# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN
+# endif // SANITIZER_INTERCEPT_TIMEDJOIN
DEFINE_REAL_PTHREAD_FUNCTIONS
@@ -522,6 +561,7 @@ INTERCEPTOR(void, _exit, int status) {
}
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_LSAN_INITED
#include "sanitizer_common/sanitizer_signal_interceptors.inc"
#endif
@@ -549,6 +589,10 @@ void InitializeInterceptors() {
LSAN_MAYBE_INTERCEPT_MALLOPT;
INTERCEPT_FUNCTION(pthread_create);
INTERCEPT_FUNCTION(pthread_join);
+ INTERCEPT_FUNCTION(pthread_detach);
+ INTERCEPT_FUNCTION(pthread_exit);
+ LSAN_MAYBE_INTERCEPT_TIMEDJOIN;
+ LSAN_MAYBE_INTERCEPT_TRYJOIN;
INTERCEPT_FUNCTION(_exit);
LSAN_MAYBE_INTERCEPT__LWP_EXIT;
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
index 63ae2c1a945b8..26aaf190ce873 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp
@@ -14,13 +14,14 @@
#if SANITIZER_LINUX || SANITIZER_NETBSD || SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN
-#include "lsan_allocator.h"
+# include "lsan_allocator.h"
+# include "lsan_thread.h"
namespace __lsan {
-static THREADLOCAL u32 current_thread_tid = kInvalidTid;
-u32 GetCurrentThread() { return current_thread_tid; }
-void SetCurrentThread(u32 tid) { current_thread_tid = tid; }
+static THREADLOCAL ThreadContextLsanBase *current_thread = nullptr;
+ThreadContextLsanBase *GetCurrentThread() { return current_thread; }
+void SetCurrentThread(ThreadContextLsanBase *tctx) { current_thread = tctx; }
static THREADLOCAL AllocatorCache allocator_cache;
AllocatorCache *GetAllocatorCache() { return &allocator_cache; }
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp b/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp
index 6964a9ba28df5..990954a8b6879 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp
@@ -67,10 +67,9 @@ typedef struct {
ALWAYS_INLINE
void lsan_register_worker_thread(int parent_tid) {
- if (GetCurrentThread() == kInvalidTid) {
+ if (GetCurrentThreadId() == kInvalidTid) {
u32 tid = ThreadCreate(parent_tid, true);
ThreadStart(tid, GetTid());
- SetCurrentThread(tid);
}
}
@@ -81,7 +80,7 @@ extern "C" void lsan_dispatch_call_block_and_release(void *block) {
VReport(2,
"lsan_dispatch_call_block_and_release(): "
"context: %p, pthread_self: %p\n",
- block, pthread_self());
+ block, (void*)pthread_self());
lsan_register_worker_thread(context->parent_tid);
// Call the original dispatcher for the block.
context->func(context->block);
@@ -101,7 +100,7 @@ extern "C" lsan_block_context_t *alloc_lsan_context(void *ctxt,
(lsan_block_context_t *)lsan_malloc(sizeof(lsan_block_context_t), stack);
lsan_ctxt->block = ctxt;
lsan_ctxt->func = func;
- lsan_ctxt->parent_tid = GetCurrentThread();
+ lsan_ctxt->parent_tid = GetCurrentThreadId();
return lsan_ctxt;
}
@@ -146,13 +145,13 @@ void dispatch_source_set_event_handler(dispatch_source_t ds,
void (^work)(void));
}
-#define GET_LSAN_BLOCK(work) \
- void (^lsan_block)(void); \
- int parent_tid = GetCurrentThread(); \
- lsan_block = ^(void) { \
- lsan_register_worker_thread(parent_tid); \
- work(); \
- }
+# define GET_LSAN_BLOCK(work) \
+ void (^lsan_block)(void); \
+ int parent_tid = GetCurrentThreadId(); \
+ lsan_block = ^(void) { \
+ lsan_register_worker_thread(parent_tid); \
+ work(); \
+ }
INTERCEPTOR(void, dispatch_async, dispatch_queue_t dq, void (^work)(void)) {
GET_LSAN_BLOCK(work);
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp
index 3c7bc15a851af..d99e1cc0105ef 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp
@@ -35,6 +35,7 @@ struct OnStartedArgs {
};
void ThreadContext::OnStarted(void *arg) {
+ ThreadContextLsanBase::OnStarted(arg);
auto args = reinterpret_cast(arg);
stack_begin_ = args->stack_begin;
stack_end_ = args->stack_end;
@@ -88,7 +89,7 @@ static void OnStackUnwind(const SignalContext &sig, const void *,
}
void LsanOnDeadlySignal(int signo, void *siginfo, void *context) {
- HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind,
+ HandleDeadlySignal(siginfo, context, GetCurrentThreadId(), &OnStackUnwind,
nullptr);
}
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
index 2b694ba5e4071..f27e36f2e9ca4 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
+++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp
@@ -24,24 +24,41 @@
namespace __lsan {
static ThreadRegistry *thread_registry;
+static ThreadArgRetval *thread_arg_retval;
+
+static Mutex mu_for_thread_context;
+static LowLevelAllocator allocator_for_thread_context;
static ThreadContextBase *CreateThreadContext(u32 tid) {
- void *mem = MmapOrDie(sizeof(ThreadContext), "ThreadContext");
- return new (mem) ThreadContext(tid);
+ Lock lock(&mu_for_thread_context);
+ return new (allocator_for_thread_context) ThreadContext(tid);
}
-void InitializeThreadRegistry() {
- static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)];
+void InitializeThreads() {
+ static ALIGNED(alignof(
+ ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)];
thread_registry =
new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext);
+
+ static ALIGNED(alignof(ThreadArgRetval)) char
+ thread_arg_retval_placeholder[sizeof(ThreadArgRetval)];
+ thread_arg_retval = new (thread_arg_retval_placeholder) ThreadArgRetval();
}
+ThreadArgRetval &GetThreadArgRetval() { return *thread_arg_retval; }
+
ThreadContextLsanBase::ThreadContextLsanBase(int tid)
: ThreadContextBase(tid) {}
+void ThreadContextLsanBase::OnStarted(void *arg) {
+ SetCurrentThread(this);
+ AllocatorThreadStart();
+}
+
void ThreadContextLsanBase::OnFinished() {
AllocatorThreadFinish();
DTLS_Destroy();
+ SetCurrentThread(nullptr);
}
u32 ThreadCreate(u32 parent_tid, bool detached, void *arg) {
@@ -51,26 +68,13 @@ u32 ThreadCreate(u32 parent_tid, bool detached, void *arg) {
void ThreadContextLsanBase::ThreadStart(u32 tid, tid_t os_id,
ThreadType thread_type, void *arg) {
thread_registry->StartThread(tid, os_id, thread_type, arg);
- SetCurrentThread(tid);
-}
-
-void ThreadFinish() {
- thread_registry->FinishThread(GetCurrentThread());
- SetCurrentThread(kInvalidTid);
}
-ThreadContext *CurrentThreadContext() {
- if (!thread_registry)
- return nullptr;
- if (GetCurrentThread() == kInvalidTid)
- return nullptr;
- // No lock needed when getting current thread.
- return (ThreadContext *)thread_registry->GetThreadLocked(GetCurrentThread());
-}
+void ThreadFinish() { thread_registry->FinishThread(GetCurrentThreadId()); }
void EnsureMainThreadIDIsCorrect() {
- if (GetCurrentThread() == kMainTid)
- CurrentThreadContext()->os_id = GetTid();
+ if (GetCurrentThreadId() == kMainTid)
+ GetCurrentThread()->os_id = GetTid();
}
///// Interface to the common LSan module. /////
@@ -79,9 +83,15 @@ void GetThreadExtraStackRangesLocked(tid_t os_id,
InternalMmapVector *ranges) {}
void GetThreadExtraStackRangesLocked(InternalMmapVector *ranges) {}
-void LockThreadRegistry() { thread_registry->Lock(); }
+void LockThreads() {
+ thread_registry->Lock();
+ thread_arg_retval->Lock();
+}
-void UnlockThreadRegistry() { thread_registry->Unlock(); }
+void UnlockThreads() {
+ thread_arg_retval->Unlock();
+ thread_registry->Unlock();
+}
ThreadRegistry *GetLsanThreadRegistryLocked() {
thread_registry->CheckLocked();
@@ -99,6 +109,10 @@ void GetRunningThreadsLocked(InternalMmapVector *threads) {
threads);
}
+void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) {
+ GetThreadArgRetval().GetAllPtrsLocked(ptrs);
+}
+
} // namespace __lsan
namespace __sanitizer {
diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.h b/system/lib/compiler-rt/lib/lsan/lsan_thread.h
index 049c7e2038017..222066ee93cd9 100644
--- a/system/lib/compiler-rt/lib/lsan/lsan_thread.h
+++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.h
@@ -14,6 +14,7 @@
#ifndef LSAN_THREAD_H
#define LSAN_THREAD_H
+#include "sanitizer_common/sanitizer_thread_arg_retval.h"
#include "sanitizer_common/sanitizer_thread_registry.h"
namespace __lsan {
@@ -21,6 +22,7 @@ namespace __lsan {
class ThreadContextLsanBase : public ThreadContextBase {
public:
explicit ThreadContextLsanBase(int tid);
+ void OnStarted(void *arg) override;
void OnFinished() override;
uptr stack_begin() { return stack_begin_; }
uptr stack_end() { return stack_end_; }
@@ -42,17 +44,21 @@ class ThreadContextLsanBase : public ThreadContextBase {
// This subclass of ThreadContextLsanBase is declared in an OS-specific header.
class ThreadContext;
-void InitializeThreadRegistry();
+void InitializeThreads();
void InitializeMainThread();
ThreadRegistry *GetLsanThreadRegistryLocked();
+ThreadArgRetval &GetThreadArgRetval();
u32 ThreadCreate(u32 tid, bool detached, void *arg = nullptr);
void ThreadFinish();
-u32 GetCurrentThread();
-void SetCurrentThread(u32 tid);
-ThreadContext *CurrentThreadContext();
+ThreadContextLsanBase *GetCurrentThread();
+inline u32 GetCurrentThreadId() {
+ ThreadContextLsanBase *ctx = GetCurrentThread();
+ return ctx ? ctx->tid : kInvalidTid;
+}
+void SetCurrentThread(ThreadContextLsanBase *tctx);
void EnsureMainThreadIDIsCorrect();
} // namespace __lsan
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
index 25a43a59f0475..03392b61503b0 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp
@@ -146,12 +146,10 @@ void *LowLevelAllocator::Allocate(uptr size) {
size = RoundUpTo(size, low_level_alloc_min_alignment);
if (allocated_end_ - allocated_current_ < (sptr)size) {
uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached());
- allocated_current_ =
- (char*)MmapOrDie(size_to_allocate, __func__);
+ allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__);
allocated_end_ = allocated_current_ + size_to_allocate;
if (low_level_alloc_callback) {
- low_level_alloc_callback((uptr)allocated_current_,
- size_to_allocate);
+ low_level_alloc_callback((uptr)allocated_current_, size_to_allocate);
}
}
CHECK(allocated_end_ - allocated_current_ >= (sptr)size);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
index 76b936ff5eaa6..0b28f86d14084 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h
@@ -62,6 +62,13 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) {
*rand_state = state;
}
+struct NoOpMapUnmapCallback {
+ void OnMap(uptr p, uptr size) const {}
+ void OnMapSecondary(uptr p, uptr size, uptr user_begin,
+ uptr user_size) const {}
+ void OnUnmap(uptr p, uptr size) const {}
+};
+
#include "sanitizer_allocator_size_class_map.h"
#include "sanitizer_allocator_stats.h"
#include "sanitizer_allocator_primary64.h"
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
index b76d36dcf5a4e..49940d9b5d505 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h
@@ -29,9 +29,9 @@ class CombinedAllocator {
LargeMmapAllocatorPtrArray,
typename PrimaryAllocator::AddressSpaceView>;
- void InitLinkerInitialized(s32 release_to_os_interval_ms) {
- stats_.InitLinkerInitialized();
- primary_.Init(release_to_os_interval_ms);
+ void InitLinkerInitialized(s32 release_to_os_interval_ms,
+ uptr heap_start = 0) {
+ primary_.Init(release_to_os_interval_ms, heap_start);
secondary_.InitLinkerInitialized();
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
index c1b27563e2fc7..de2b271fb0ed9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h
@@ -21,8 +21,12 @@ extern "C" {
SANITIZER_INTERFACE_ATTRIBUTE
uptr __sanitizer_get_estimated_allocated_size(uptr size);
SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_get_ownership(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE const void *__sanitizer_get_allocated_begin(
+ const void *p);
SANITIZER_INTERFACE_ATTRIBUTE uptr
__sanitizer_get_allocated_size(const void *p);
+SANITIZER_INTERFACE_ATTRIBUTE uptr
+__sanitizer_get_allocated_size_fast(const void *p);
SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes();
SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size();
SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes();
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h
index 38994736877ac..62523c7ae187c 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h
@@ -51,7 +51,6 @@ void InternalFree(void *p, InternalAllocatorCache *cache = nullptr);
void InternalAllocatorLock();
void InternalAllocatorUnlock();
InternalAllocator *internal_allocator();
-
} // namespace __sanitizer
#endif // SANITIZER_ALLOCATOR_INTERNAL_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
index f2471efced613..52fe3fe3d15bd 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h
@@ -353,7 +353,7 @@ class SizeClassAllocator32 {
DCHECK_GT(max_count, 0);
TransferBatch *b = nullptr;
constexpr uptr kShuffleArraySize = 48;
- uptr shuffle_array[kShuffleArraySize];
+ UNINITIALIZED uptr shuffle_array[kShuffleArraySize];
uptr count = 0;
for (uptr i = region; i < region + n_chunks * size; i += size) {
shuffle_array[count++] = i;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 66ba71d325dad..fa43ac50c61e4 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -635,8 +635,8 @@ class SizeClassAllocator64 {
return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg;
}
uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; }
- // kRegionSize must be >= 2^32.
- COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2)));
+ // kRegionSize should be able to satisfy the largest size class.
+ static_assert(kRegionSize >= SizeClassMap::kMaxSize);
// kRegionSize must be <= 2^36, see CompactPtrT.
COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4)));
// Call mmap for user memory with at least this size.
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
index 1576455556044..0607819e7ef7c 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h
@@ -82,7 +82,7 @@ class LargeMmapAllocator {
InitLinkerInitialized();
}
- void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) {
+ void *Allocate(AllocatorStats *stat, const uptr size, uptr alignment) {
CHECK(IsPowerOfTwo(alignment));
uptr map_size = RoundUpMapSize(size);
if (alignment > page_size_)
@@ -99,11 +99,11 @@ class LargeMmapAllocator {
if (!map_beg)
return nullptr;
CHECK(IsAligned(map_beg, page_size_));
- MapUnmapCallback().OnMap(map_beg, map_size);
uptr map_end = map_beg + map_size;
uptr res = map_beg + page_size_;
if (res & (alignment - 1)) // Align.
res += alignment - (res & (alignment - 1));
+ MapUnmapCallback().OnMapSecondary(map_beg, map_size, res, size);
CHECK(IsAligned(res, alignment));
CHECK(IsAligned(res, page_size_));
CHECK_GE(res + size, map_beg);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h
index 6f14e3863c31c..ae4dac9c8c962 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h
@@ -25,19 +25,13 @@ typedef uptr AllocatorStatCounters[AllocatorStatCount];
// Per-thread stats, live in per-thread cache.
class AllocatorStats {
public:
- void Init() {
- internal_memset(this, 0, sizeof(*this));
- }
- void InitLinkerInitialized() {}
-
+ void Init() { internal_memset(this, 0, sizeof(*this)); }
void Add(AllocatorStat i, uptr v) {
- v += atomic_load(&stats_[i], memory_order_relaxed);
- atomic_store(&stats_[i], v, memory_order_relaxed);
+ atomic_fetch_add(&stats_[i], v, memory_order_relaxed);
}
void Sub(AllocatorStat i, uptr v) {
- v = atomic_load(&stats_[i], memory_order_relaxed) - v;
- atomic_store(&stats_[i], v, memory_order_relaxed);
+ atomic_fetch_sub(&stats_[i], v, memory_order_relaxed);
}
void Set(AllocatorStat i, uptr v) {
@@ -58,17 +52,13 @@ class AllocatorStats {
// Global stats, used for aggregation and querying.
class AllocatorGlobalStats : public AllocatorStats {
public:
- void InitLinkerInitialized() {
- next_ = this;
- prev_ = this;
- }
void Init() {
internal_memset(this, 0, sizeof(*this));
- InitLinkerInitialized();
}
void Register(AllocatorStats *s) {
SpinMutexLock l(&mu_);
+ LazyInit();
s->next_ = next_;
s->prev_ = this;
next_->prev_ = s;
@@ -87,7 +77,7 @@ class AllocatorGlobalStats : public AllocatorStats {
internal_memset(s, 0, AllocatorStatCount * sizeof(uptr));
SpinMutexLock l(&mu_);
const AllocatorStats *stats = this;
- for (;;) {
+ for (; stats;) {
for (int i = 0; i < AllocatorStatCount; i++)
s[i] += stats->Get(AllocatorStat(i));
stats = stats->next_;
@@ -100,6 +90,13 @@ class AllocatorGlobalStats : public AllocatorStats {
}
private:
+ void LazyInit() {
+ if (!next_) {
+ next_ = this;
+ prev_ = this;
+ }
+ }
+
mutable StaticSpinMutex mu_;
};
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h
new file mode 100644
index 0000000000000..28d125383da41
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h
@@ -0,0 +1,123 @@
+//===-- sanitizer_array_ref.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_ARRAY_REF_H
+#define SANITIZER_ARRAY_REF_H
+
+#include "sanitizer_internal_defs.h"
+
+namespace __sanitizer {
+
+/// ArrayRef - Represent a constant reference to an array (0 or more elements
+/// consecutively in memory), i.e. a start pointer and a length. It allows
+/// various APIs to take consecutive elements easily and conveniently.
+///
+/// This class does not own the underlying data, it is expected to be used in
+/// situations where the data resides in some other buffer, whose lifetime
+/// extends past that of the ArrayRef. For this reason, it is not in general
+/// safe to store an ArrayRef.
+///
+/// This is intended to be trivially copyable, so it should be passed by
+/// value.
+template
+class ArrayRef {
+ public:
+ constexpr ArrayRef() {}
+ constexpr ArrayRef(const T *begin, const T *end) : begin_(begin), end_(end) {
+ DCHECK(empty() || begin);
+ }
+ constexpr ArrayRef(const T *data, uptr length)
+ : ArrayRef(data, data + length) {}
+ template
+ constexpr ArrayRef(const T (&src)[N]) : ArrayRef(src, src + N) {}
+ template
+ constexpr ArrayRef(const C &src)
+ : ArrayRef(src.data(), src.data() + src.size()) {}
+ ArrayRef(const T &one_elt) : ArrayRef(&one_elt, &one_elt + 1) {}
+
+ const T *data() const { return empty() ? nullptr : begin_; }
+
+ const T *begin() const { return begin_; }
+ const T *end() const { return end_; }
+
+ bool empty() const { return begin_ == end_; }
+
+ uptr size() const { return end_ - begin_; }
+
+ /// equals - Check for element-wise equality.
+ bool equals(ArrayRef rhs) const {
+ if (size() != rhs.size())
+ return false;
+ auto r = rhs.begin();
+ for (auto &l : *this) {
+ if (!(l == *r))
+ return false;
+ ++r;
+ }
+ return true;
+ }
+
+ /// slice(n, m) - Chop off the first N elements of the array, and keep M
+ /// elements in the array.
+ ArrayRef slice(uptr N, uptr M) const {
+ DCHECK_LE(N + M, size());
+ return ArrayRef(data() + N, M);
+ }
+
+ /// slice(n) - Chop off the first N elements of the array.
+ ArrayRef slice(uptr N) const { return slice(N, size() - N); }
+
+ /// Drop the first \p N elements of the array.
+ ArrayRef drop_front(uptr N = 1) const {
+ DCHECK_GE(size(), N);
+ return slice(N, size() - N);
+ }
+
+ /// Drop the last \p N elements of the array.
+ ArrayRef drop_back(uptr N = 1) const {
+ DCHECK_GE(size(), N);
+ return slice(0, size() - N);
+ }
+
+ /// Return a copy of *this with only the first \p N elements.
+ ArrayRef take_front(uptr N = 1) const {
+ if (N >= size())
+ return *this;
+ return drop_back(size() - N);
+ }
+
+ /// Return a copy of *this with only the last \p N elements.
+ ArrayRef take_back(uptr N = 1) const {
+ if (N >= size())
+ return *this;
+ return drop_front(size() - N);
+ }
+
+ const T &operator[](uptr index) const {
+ DCHECK_LT(index, size());
+ return begin_[index];
+ }
+
+ private:
+ const T *begin_ = nullptr;
+ const T *end_ = nullptr;
+};
+
+template
+inline bool operator==(ArrayRef lhs, ArrayRef rhs) {
+ return lhs.equals(rhs);
+}
+
+template
+inline bool operator!=(ArrayRef lhs, ArrayRef rhs) {
+ return !(lhs == rhs);
+}
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_ARRAY_REF_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
index 9ebba91da73ff..3c9bbdc9678b0 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h
@@ -42,13 +42,57 @@
# define CFI_RESTORE(reg)
#endif
+#if defined(__x86_64__) || defined(__i386__) || defined(__sparc__)
+# define ASM_TAIL_CALL jmp
+#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \
+ defined(__powerpc__) || defined(__loongarch_lp64)
+# define ASM_TAIL_CALL b
+#elif defined(__s390__)
+# define ASM_TAIL_CALL jg
+#elif defined(__riscv)
+# define ASM_TAIL_CALL tail
+#endif
+
+#if defined(__ELF__) && defined(__x86_64__) || defined(__i386__) || \
+ defined(__riscv)
+# define ASM_PREEMPTIBLE_SYM(sym) sym@plt
+#else
+# define ASM_PREEMPTIBLE_SYM(sym) sym
+#endif
+
#if !defined(__APPLE__)
# define ASM_HIDDEN(symbol) .hidden symbol
# define ASM_TYPE_FUNCTION(symbol) .type symbol, %function
# define ASM_SIZE(symbol) .size symbol, .-symbol
# define ASM_SYMBOL(symbol) symbol
# define ASM_SYMBOL_INTERCEPTOR(symbol) symbol
-# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
+# if defined(__i386__) || defined(__powerpc__) || defined(__s390__) || \
+ defined(__sparc__)
+// For details, see interception.h
+# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol
+# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
+ .weak symbol; \
+ .set symbol, ASM_WRAPPER_NAME(name)
+# define ASM_INTERCEPTOR_TRAMPOLINE(name)
+# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 0
+# else // Architecture supports interceptor trampoline
+// Keep trampoline implementation in sync with interception/interception.h
+# define ASM_WRAPPER_NAME(symbol) ___interceptor_##symbol
+# define ASM_TRAMPOLINE_ALIAS(symbol, name) \
+ .weak symbol; \
+ .set symbol, __interceptor_trampoline_##name
+# define ASM_INTERCEPTOR_TRAMPOLINE(name) \
+ .weak __interceptor_##name; \
+ .set __interceptor_##name, ASM_WRAPPER_NAME(name); \
+ .globl __interceptor_trampoline_##name; \
+ ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \
+ __interceptor_trampoline_##name: \
+ CFI_STARTPROC; \
+ ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \
+ CFI_ENDPROC; \
+ ASM_SIZE(__interceptor_trampoline_##name)
+# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1
+# endif // Architecture supports interceptor trampoline
#else
# define ASM_HIDDEN(symbol)
# define ASM_TYPE_FUNCTION(symbol)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
index 82236453157fa..79b7748b8f6e8 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp
@@ -61,6 +61,26 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
UNREACHABLE("unable to mmap");
}
+void NORETURN ReportMunmapFailureAndDie(void *addr, uptr size, error_t err,
+ bool raw_report) {
+ static int recursion_count;
+ if (raw_report || recursion_count) {
+ // If raw report is requested or we went into recursion just die. The
+ // Report() and CHECK calls below may call munmap recursively and fail.
+ RawWrite("ERROR: Failed to munmap\n");
+ Die();
+ }
+ recursion_count++;
+ Report(
+ "ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p (error "
+ "code: %d)\n",
+ SanitizerToolName, size, size, addr, err);
+#if !SANITIZER_GO
+ DumpProcessMap();
+#endif
+ UNREACHABLE("unable to unmmap");
+}
+
typedef bool UptrComparisonFunction(const uptr &a, const uptr &b);
typedef bool U32ComparisonFunction(const u32 &a, const u32 &b);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
index b462e388c232e..e7e4b8cb506db 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h
@@ -117,6 +117,7 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
// unaccessible memory.
bool MprotectNoAccess(uptr addr, uptr size);
bool MprotectReadOnly(uptr addr, uptr size);
+bool MprotectReadWrite(uptr addr, uptr size);
void MprotectMallocZones(void *addr, int prot);
@@ -211,6 +212,7 @@ class LowLevelAllocator {
public:
// Requires an external lock.
void *Allocate(uptr size);
+
private:
char *allocated_end_;
char *allocated_current_;
@@ -315,6 +317,8 @@ CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2);
void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type,
const char *mmap_type, error_t err,
bool raw_report = false);
+void NORETURN ReportMunmapFailureAndDie(void *ptr, uptr size, error_t err,
+ bool raw_report = false);
// Returns true if the platform-specific error reported is an OOM error.
bool ErrorIsOOM(error_t err);
@@ -516,8 +520,8 @@ class InternalMmapVectorNoCtor {
return data_[i];
}
void push_back(const T &element) {
- CHECK_LE(size_, capacity());
- if (size_ == capacity()) {
+ if (UNLIKELY(size_ >= capacity())) {
+ CHECK_EQ(size_, capacity());
uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1);
Realloc(new_capacity);
}
@@ -577,7 +581,7 @@ class InternalMmapVectorNoCtor {
}
private:
- void Realloc(uptr new_capacity) {
+ NOINLINE void Realloc(uptr new_capacity) {
CHECK_GT(new_capacity, 0);
CHECK_LE(size_, new_capacity);
uptr new_capacity_bytes =
@@ -793,7 +797,11 @@ inline const char *ModuleArchToString(ModuleArch arch) {
return "";
}
+#if SANITIZER_APPLE
+const uptr kModuleUUIDSize = 16;
+#else
const uptr kModuleUUIDSize = 32;
+#endif
const uptr kMaxSegName = 16;
// Represents a binary loaded into virtual memory (e.g. this can be an
@@ -1076,20 +1084,6 @@ inline u32 GetNumberOfCPUsCached() {
return NumberOfCPUsCached;
}
-template
-class ArrayRef {
- public:
- ArrayRef() {}
- ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {}
-
- T *begin() { return begin_; }
- T *end() { return end_; }
-
- private:
- T *begin_ = nullptr;
- T *end_ = nullptr;
-};
-
} // namespace __sanitizer
inline void *operator new(__sanitizer::operator_new_size_type size,
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index e999239549ccc..0e563fa12022a 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -26,10 +26,8 @@
// COMMON_INTERCEPTOR_SET_PTHREAD_NAME
// COMMON_INTERCEPTOR_HANDLE_RECVMSG
// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
-// COMMON_INTERCEPTOR_MEMSET_IMPL
-// COMMON_INTERCEPTOR_MEMMOVE_IMPL
-// COMMON_INTERCEPTOR_MEMCPY_IMPL
// COMMON_INTERCEPTOR_MMAP_IMPL
+// COMMON_INTERCEPTOR_MUNMAP_IMPL
// COMMON_INTERCEPTOR_COPY_STRING
// COMMON_INTERCEPTOR_STRNDUP_IMPL
// COMMON_INTERCEPTOR_STRERROR
@@ -198,15 +196,6 @@ extern const short *_tolower_tab_;
#define wait4 __wait4_time64
#endif
-// Platform-specific options.
-#if SANITIZER_APPLE
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
-#elif SANITIZER_WINDOWS64
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
-#else
-#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
-#endif // SANITIZER_APPLE
-
#ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE
#define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {}
#endif
@@ -302,53 +291,17 @@ extern const short *_tolower_tab_;
COMMON_INTERCEPT_FUNCTION(fn)
#endif
-#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
-#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
- { \
- if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
- return internal_memset(dst, v, size); \
- COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
- if (common_flags()->intercept_intrin) \
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
- return REAL(memset)(dst, v, size); \
- }
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
-#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
- { \
- if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
- return internal_memmove(dst, src, size); \
- COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
- if (common_flags()->intercept_intrin) { \
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
- } \
- return REAL(memmove)(dst, src, size); \
- }
-#endif
-
-#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
-#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
- { \
- if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
- return internal_memmove(dst, src, size); \
- } \
- COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
- if (common_flags()->intercept_intrin) { \
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
- COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
- } \
- return REAL(memcpy)(dst, src, size); \
- }
-#endif
-
#ifndef COMMON_INTERCEPTOR_MMAP_IMPL
#define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
off) \
{ return REAL(mmap)(addr, sz, prot, flags, fd, off); }
#endif
+#ifndef COMMON_INTERCEPTOR_MUNMAP_IMPL
+#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \
+ { return REAL(munmap)(addr, sz); }
+#endif
+
#ifndef COMMON_INTERCEPTOR_COPY_STRING
#define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {}
#endif
@@ -841,57 +794,6 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) {
#define INIT_STRPBRK
#endif
-#if SANITIZER_INTERCEPT_MEMSET
-INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
-}
-
-#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
-#else
-#define INIT_MEMSET
-#endif
-
-#if SANITIZER_INTERCEPT_MEMMOVE
-INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-}
-
-#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
-#else
-#define INIT_MEMMOVE
-#endif
-
-#if SANITIZER_INTERCEPT_MEMCPY
-INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
- // On OS X, calling internal_memcpy here will cause memory corruptions,
- // because memcpy and memmove are actually aliases of the same
- // implementation. We need to use internal_memmove here.
- // N.B.: If we switch this to internal_ we'll have to use internal_memmove
- // due to memcpy being an alias of memmove on OS X.
- void *ctx;
-#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
- COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
-#else
- COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
-#endif
-}
-
-#define INIT_MEMCPY \
- do { \
- if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
- COMMON_INTERCEPT_FUNCTION(memcpy); \
- } else { \
- ASSIGN_REAL(memcpy, memmove); \
- } \
- CHECK(REAL(memcpy)); \
- } while (false)
-
-#else
-#define INIT_MEMCPY
-#endif
-
#if SANITIZER_INTERCEPT_MEMCMP
DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc,
const void *s1, const void *s2, uptr n,
@@ -1350,7 +1252,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3,
char *name = (char *)arg5;
COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1);
}
- int res = REAL(prctl(option, arg2, arg3, arg4, arg5));
+ int res = REAL(prctl)(option, arg2, arg3, arg4, arg5);
if (option == PR_SET_NAME) {
char buff[16];
internal_strncpy(buff, (char *)arg2, 15);
@@ -1589,6 +1491,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap)
INTERCEPTOR(int, __isoc99_vfscanf, void *stream, const char *format, va_list ap)
VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap)
+
+INTERCEPTOR(int, __isoc23_vscanf, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc23_vscanf, false, format, ap)
+
+INTERCEPTOR(int, __isoc23_vsscanf, const char *str, const char *format,
+ va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc23_vsscanf, false, str, format, ap)
+
+INTERCEPTOR(int, __isoc23_vfscanf, void *stream, const char *format, va_list ap)
+VSCANF_INTERCEPTOR_IMPL(__isoc23_vfscanf, false, stream, format, ap)
#endif // SANITIZER_INTERCEPT_ISOC99_SCANF
INTERCEPTOR(int, scanf, const char *format, ...)
@@ -1609,6 +1521,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format)
INTERCEPTOR(int, __isoc99_sscanf, const char *str, const char *format, ...)
FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
+
+INTERCEPTOR(int, __isoc23_scanf, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc23_scanf, __isoc23_vscanf, format)
+
+INTERCEPTOR(int, __isoc23_fscanf, void *stream, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc23_fscanf, __isoc23_vfscanf, stream, format)
+
+INTERCEPTOR(int, __isoc23_sscanf, const char *str, const char *format, ...)
+FORMAT_INTERCEPTOR_IMPL(__isoc23_sscanf, __isoc23_vsscanf, str, format)
#endif
#endif
@@ -1632,7 +1553,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format)
COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \
COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \
COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \
- COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf);
+ COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_scanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_sscanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_fscanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_vscanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_vsscanf); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_vfscanf);
#else
#define INIT_ISOC99_SCANF
#endif
@@ -3416,7 +3343,8 @@ INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) {
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
__sanitizer_dirent *res = REAL(readdir)(dirp);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
+ if (res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
return res;
}
@@ -3431,7 +3359,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry,
if (!res) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
if (*result)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result));
}
return res;
}
@@ -3452,7 +3380,8 @@ INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) {
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
__sanitizer_dirent64 *res = REAL(readdir64)(dirp);
- if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen);
+ if (res)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res));
return res;
}
@@ -3467,7 +3396,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry,
if (!res) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result));
if (*result)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result));
}
return res;
}
@@ -3635,30 +3564,26 @@ UNUSED static inline void StrtolFixAndCheck(void *ctx, const char *nptr,
(real_endptr - nptr) + 1 : 0);
}
-
#if SANITIZER_INTERCEPT_STRTOIMAX
-INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
+template
+static ALWAYS_INLINE auto StrtoimaxImpl(void *ctx, Fn real, const char *nptr,
+ char **endptr, int base)
+ -> decltype(real(nullptr, nullptr, 0)) {
char *real_endptr;
- INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base);
+ auto res = real(nptr, &real_endptr, base);
StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
return res;
}
+INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base);
+ return StrtoimaxImpl(ctx, REAL(strtoimax), nptr, endptr, base);
+}
INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- char *real_endptr;
- UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base);
- StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base);
- return res;
+ return StrtoimaxImpl(ctx, REAL(strtoumax), nptr, endptr, base);
}
#define INIT_STRTOIMAX \
@@ -3668,6 +3593,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) {
#define INIT_STRTOIMAX
#endif
+#if SANITIZER_INTERCEPT_STRTOIMAX && SANITIZER_GLIBC
+INTERCEPTOR(INTMAX_T, __isoc23_strtoimax, const char *nptr, char **endptr, int base) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoimax, nptr, endptr, base);
+ return StrtoimaxImpl(ctx, REAL(__isoc23_strtoimax), nptr, endptr, base);
+}
+INTERCEPTOR(UINTMAX_T, __isoc23_strtoumax, const char *nptr, char **endptr, int base) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoumax, nptr, endptr, base);
+ return StrtoimaxImpl(ctx, REAL(__isoc23_strtoumax), nptr, endptr, base);
+}
+
+# define INIT_STRTOIMAX_C23 \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_strtoimax); \
+ COMMON_INTERCEPT_FUNCTION(__isoc23_strtoumax);
+#else
+# define INIT_STRTOIMAX_C23
+#endif
+
#if SANITIZER_INTERCEPT_MBSTOWCS
INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) {
void *ctx;
@@ -4039,7 +3983,7 @@ static THREADLOCAL scandir_compar_f scandir_compar;
static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
return scandir_filter(dir);
}
@@ -4047,9 +3991,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a,
const struct __sanitizer_dirent **b) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a));
COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b));
return scandir_compar(a, b);
}
@@ -4073,7 +4017,7 @@ INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist,
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res);
for (int i = 0; i < res; ++i)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
- (*namelist)[i]->d_reclen);
+ __sanitizer_dirsiz((*namelist)[i]));
}
return res;
}
@@ -4092,7 +4036,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar;
static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(1);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir));
return scandir64_filter(dir);
}
@@ -4100,9 +4044,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a,
const struct __sanitizer_dirent64 **b) {
COMMON_INTERCEPTOR_UNPOISON_PARAM(2);
COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a));
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a));
COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b));
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen);
+ COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b));
return scandir64_compar(a, b);
}
@@ -4127,7 +4071,7 @@ INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist,
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res);
for (int i = 0; i < res; ++i)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i],
- (*namelist)[i]->d_reclen);
+ __sanitizer_dirsiz((*namelist)[i]));
}
return res;
}
@@ -4404,12 +4348,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set,
INTERCEPTOR(int, backtrace, void **buffer, int size) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size);
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
- int res = REAL(backtrace)(buffer, size);
- if (res && buffer)
+ // 'buffer' might be freed memory, hence it is unsafe to directly call
+ // REAL(backtrace)(buffer, size). Instead, we use our own known-good
+ // scratch buffer.
+ void **scratch = (void**)InternalAlloc(sizeof(void*) * size);
+ int res = REAL(backtrace)(scratch, size);
+ if (res && buffer) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer));
+ internal_memcpy(buffer, scratch, res * sizeof(*buffer));
+ }
+ InternalFree(scratch);
return res;
}
@@ -4418,9 +4366,8 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) {
COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size);
if (buffer && size)
COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer));
- // FIXME: under ASan the call below may write to freed memory and corrupt
- // its metadata. See
- // https://github.com/google/sanitizers/issues/321.
+ // The COMMON_INTERCEPTOR_READ_RANGE above ensures that 'buffer' is
+ // valid for reading.
char **res = REAL(backtrace_symbols)(buffer, size);
if (res && size) {
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res));
@@ -4453,7 +4400,7 @@ INTERCEPTOR(void, _exit, int status) {
#if SANITIZER_INTERCEPT___LIBC_MUTEX
INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate)
-ALIAS(WRAPPER_NAME(pthread_setcancelstate));
+ALIAS(WRAP(pthread_setcancelstate));
#define INIT___LIBC_THR_SETCANCELSTATE \
COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate)
@@ -5484,9 +5431,7 @@ INTERCEPTOR(void *, __tls_get_addr, void *arg) {
// On PowerPC, we also need to intercept __tls_get_addr_opt, which has
// mostly the same semantics as __tls_get_addr, but its presence enables
// some optimizations in linker (which are safe to ignore here).
-extern "C" __attribute__((alias("__interceptor___tls_get_addr"),
- visibility("default")))
-void *__tls_get_addr_opt(void *arg);
+INTERCEPTOR(void *, __tls_get_addr_opt, void *arg) ALIAS(WRAP(__tls_get_addr));
#endif
#else // SANITIZER_S390
// On s390, we have to intercept two functions here:
@@ -5520,21 +5465,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) {
#if SANITIZER_S390 && \
(SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET)
-extern "C" uptr __tls_get_offset(void *arg);
-extern "C" uptr __interceptor___tls_get_offset(void *arg);
// We need a hidden symbol aliasing the above, so that we can jump
// directly to it from the assembly below.
-extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"),
- visibility("hidden")))
-uptr __tls_get_addr_hidden(void *arg);
+extern "C" __attribute__((visibility("hidden"))) uptr __tls_get_addr_hidden(
+ void *arg) ALIAS(WRAP(__tls_get_addr_internal));
+extern "C" uptr __tls_get_offset(void *arg);
+extern "C" uptr TRAMPOLINE(__tls_get_offset)(void *arg);
+extern "C" uptr WRAP(__tls_get_offset)(void *arg);
// Now carefully intercept __tls_get_offset.
asm(
".text\n"
// The __intercept_ version has to exist, so that gen_dynamic_list.py
// exports our symbol.
".weak __tls_get_offset\n"
- ".type __tls_get_offset, @function\n"
- "__tls_get_offset:\n"
+ ".set __tls_get_offset, __interceptor___tls_get_offset\n"
".global __interceptor___tls_get_offset\n"
".type __interceptor___tls_get_offset, @function\n"
"__interceptor___tls_get_offset:\n"
@@ -5763,8 +5707,10 @@ INTERCEPTOR(int, capget, void *hdrp, void *datap) {
// its metadata. See
// https://github.com/google/sanitizers/issues/321.
int res = REAL(capget)(hdrp, datap);
- if (res == 0 && datap)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz);
+ if (res == 0 && datap) {
+ unsigned datasz = __user_cap_data_struct_sz(hdrp);
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, datasz);
+ }
// We can also return -1 and write to hdrp->version if the version passed in
// hdrp->version is unsupported. But that's not a trivial condition to check,
// and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent.
@@ -5775,8 +5721,10 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) {
COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap);
if (hdrp)
COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz);
- if (datap)
- COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, __user_cap_data_struct_sz);
+ if (datap) {
+ unsigned datasz = __user_cap_data_struct_sz(hdrp);
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, datasz);
+ }
return REAL(capset)(hdrp, datap);
}
#define INIT_CAPGET \
@@ -5786,105 +5734,6 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) {
#define INIT_CAPGET
#endif
-#if SANITIZER_INTERCEPT_AEABI_MEM
-INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
-}
-
-// Note the argument order.
-INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-
-#define INIT_AEABI_MEM \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
- COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
-#else
-#define INIT_AEABI_MEM
-#endif // SANITIZER_INTERCEPT_AEABI_MEM
-
-#if SANITIZER_INTERCEPT___BZERO
-INTERCEPTOR(void *, __bzero, void *block, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
-#else
-#define INIT___BZERO
-#endif // SANITIZER_INTERCEPT___BZERO
-
-#if SANITIZER_INTERCEPT_BZERO
-INTERCEPTOR(void *, bzero, void *block, uptr size) {
- void *ctx;
- COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
-}
-#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
-#else
-#define INIT_BZERO
-#endif // SANITIZER_INTERCEPT_BZERO
-
#if SANITIZER_INTERCEPT_FTIME
INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) {
void *ctx;
@@ -7169,6 +7018,7 @@ INTERCEPTOR(int, mprobe, void *ptr) {
}
#endif
+#if SANITIZER_INTERCEPT_WCSLEN
INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
void *ctx;
COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s);
@@ -7187,6 +7037,9 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) {
#define INIT_WCSLEN \
COMMON_INTERCEPT_FUNCTION(wcslen); \
COMMON_INTERCEPT_FUNCTION(wcsnlen);
+#else
+#define INIT_WCSLEN
+#endif
#if SANITIZER_INTERCEPT_WCSCAT
INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) {
@@ -7595,6 +7448,14 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd,
COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off);
}
+INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) {
+ void *ctx;
+ if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED)
+ return (int)internal_munmap(addr, sz);
+ COMMON_INTERCEPTOR_ENTER(ctx, munmap, addr, sz);
+ COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz);
+}
+
INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
void *ctx;
if (common_flags()->detect_write_exec)
@@ -7607,6 +7468,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) {
}
#define INIT_MMAP \
COMMON_INTERCEPT_FUNCTION(mmap); \
+ COMMON_INTERCEPT_FUNCTION(munmap); \
COMMON_INTERCEPT_FUNCTION(mprotect);
#else
#define INIT_MMAP
@@ -7744,8 +7606,7 @@ static void write_protoent(void *ctx, struct __sanitizer_protoent *p) {
for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, internal_strlen(*pp) + 1);
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases,
- pp_size * sizeof(char **));
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *));
}
INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) {
@@ -7851,8 +7712,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) {
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
- nn_size * sizeof(char **));
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
}
return n;
}
@@ -7873,8 +7733,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyname, const char *name) {
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
- nn_size * sizeof(char **));
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
}
return n;
}
@@ -7893,8 +7752,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) {
for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size)
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1);
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases,
- nn_size * sizeof(char **));
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *));
}
return n;
}
@@ -10086,41 +9944,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) {
#define INIT_GETRANDOM
#endif
-#if SANITIZER_INTERCEPT_CRYPT
-INTERCEPTOR(char *, crypt, char *key, char *salt) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
- char *res = REAL(crypt)(key, salt);
- if (res != nullptr)
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
- return res;
-}
-#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt);
-#else
-#define INIT_CRYPT
-#endif
-
-#if SANITIZER_INTERCEPT_CRYPT_R
-INTERCEPTOR(char *, crypt_r, char *key, char *salt, void *data) {
- void *ctx;
- COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1);
- COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1);
- char *res = REAL(crypt_r)(key, salt, data);
- if (res != nullptr) {
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data,
- __sanitizer::struct_crypt_data_sz);
- COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1);
- }
- return res;
-}
-#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r);
-#else
-#define INIT_CRYPT_R
-#endif
-
#if SANITIZER_INTERCEPT_GETENTROPY
INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) {
void *ctx;
@@ -10371,14 +10194,52 @@ INTERCEPTOR(void, hexdump, const void *ptr, int length, const char *header, int
#define INIT_HEXDUMP
#endif
+#if SANITIZER_INTERCEPT_ARGP_PARSE
+INTERCEPTOR(int, argp_parse, const struct argp *argp, int argc, char **argv,
+ unsigned flags, int *arg_index, void *input) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, argp_parse, argp, argc, argv, flags, arg_index,
+ input);
+ for (int i = 0; i < argc; i++)
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, argv[i], internal_strlen(argv[i]) + 1);
+ int res = REAL(argp_parse)(argp, argc, argv, flags, arg_index, input);
+ if (!res && arg_index)
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg_index, sizeof(int));
+ return res;
+}
+
+#define INIT_ARGP_PARSE COMMON_INTERCEPT_FUNCTION(argp_parse);
+#else
+#define INIT_ARGP_PARSE
+#endif
+
+#if SANITIZER_INTERCEPT_CPUSET_GETAFFINITY
+INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T cpusetsize, __sanitizer_cpuset_t *mask) {
+ void *ctx;
+ COMMON_INTERCEPTOR_ENTER(ctx, cpuset_getaffinity, level, which, id, cpusetsize, mask);
+ int res = REAL(cpuset_getaffinity)(level, which, id, cpusetsize, mask);
+ if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize);
+ return res;
+}
+#define INIT_CPUSET_GETAFFINITY COMMON_INTERCEPT_FUNCTION(cpuset_getaffinity);
+#else
+#define INIT_CPUSET_GETAFFINITY
+#endif
+
#include "sanitizer_common_interceptors_netbsd_compat.inc"
+namespace __sanitizer {
+void InitializeMemintrinsicInterceptors();
+} // namespace __sanitizer
+
static void InitializeCommonInterceptors() {
#if SI_POSIX
static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1];
interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap();
#endif
+ __sanitizer::InitializeMemintrinsicInterceptors();
+
INIT_MMAP;
INIT_MMAP64;
INIT_TEXTDOMAIN;
@@ -10400,9 +10261,6 @@ static void InitializeCommonInterceptors() {
INIT_STRPBRK;
INIT_STRXFRM;
INIT___STRXFRM_L;
- INIT_MEMSET;
- INIT_MEMMOVE;
- INIT_MEMCPY;
INIT_MEMCHR;
INIT_MEMCMP;
INIT_BCMP;
@@ -10486,6 +10344,7 @@ static void InitializeCommonInterceptors() {
INIT_GETCWD;
INIT_GET_CURRENT_DIR_NAME;
INIT_STRTOIMAX;
+ INIT_STRTOIMAX_C23;
INIT_MBSTOWCS;
INIT_MBSNRTOWCS;
INIT_WCSTOMBS;
@@ -10574,9 +10433,6 @@ static void InitializeCommonInterceptors() {
INIT_GETIFADDRS;
INIT_IF_INDEXTONAME;
INIT_CAPGET;
- INIT_AEABI_MEM;
- INIT___BZERO;
- INIT_BZERO;
INIT_FTIME;
INIT_XDR;
INIT_XDRREC_LINUX;
@@ -10679,8 +10535,6 @@ static void InitializeCommonInterceptors() {
INIT_GETUSERSHELL;
INIT_SL_INIT;
INIT_GETRANDOM;
- INIT_CRYPT;
- INIT_CRYPT_R;
INIT_GETENTROPY;
INIT_QSORT;
INIT_QSORT_R;
@@ -10690,6 +10544,8 @@ static void InitializeCommonInterceptors() {
INIT_UNAME;
INIT___XUNAME;
INIT_HEXDUMP;
+ INIT_ARGP_PARSE;
+ INIT_CPUSET_GETAFFINITY;
INIT___PRINTF_CHK;
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
index 220abb89c3beb..24485900644b3 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc
@@ -340,11 +340,19 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc,
size = 0;
}
COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size);
- // For %ms/%mc, write the allocated output buffer as well.
+ // For %mc/%mC/%ms/%m[/%mS, write the allocated output buffer as well.
if (dir.allocate) {
- char *buf = *(char **)argp;
- if (buf)
- COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1);
+ if (char *buf = *(char **)argp) {
+ if (dir.convSpecifier == 'c')
+ size = 1;
+ else if (dir.convSpecifier == 'C')
+ size = sizeof(wchar_t);
+ else if (dir.convSpecifier == 'S')
+ size = (internal_wcslen((wchar_t *)buf) + 1) * sizeof(wchar_t);
+ else // 's' or '['
+ size = internal_strlen(buf) + 1;
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size);
+ }
}
}
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
new file mode 100644
index 0000000000000..52e489d02cda8
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc
@@ -0,0 +1,244 @@
+//===-- sanitizer_common_interceptors_memintrinsics.inc ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Memintrinsic function interceptors for tools like AddressSanitizer,
+// ThreadSanitizer, MemorySanitizer, etc.
+//
+// These interceptors are part of the common interceptors, but separated out so
+// that implementations may add them, if necessary, to a separate source file
+// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top.
+//
+// This file should be included into the tool's memintrinsic interceptor file,
+// which has to define its own macros:
+// COMMON_INTERCEPTOR_ENTER
+// COMMON_INTERCEPTOR_READ_RANGE
+// COMMON_INTERCEPTOR_WRITE_RANGE
+// COMMON_INTERCEPTOR_MEMSET_IMPL
+// COMMON_INTERCEPTOR_MEMMOVE_IMPL
+// COMMON_INTERCEPTOR_MEMCPY_IMPL
+// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED
+//===----------------------------------------------------------------------===//
+
+#ifdef SANITIZER_REDEFINE_BUILTINS_H
+#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file"
+#endif
+
+#include "interception/interception.h"
+#include "sanitizer_platform_interceptors.h"
+
+// Platform-specific options.
+#if SANITIZER_APPLE
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#elif SANITIZER_WINDOWS64
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0
+#else
+#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1
+#endif // SANITIZER_APPLE
+
+#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL
+#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \
+ { \
+ if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
+ return internal_memset(dst, v, size); \
+ COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \
+ if (common_flags()->intercept_intrin) \
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
+ return REAL(memset)(dst, v, size); \
+ }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL
+#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \
+ { \
+ if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \
+ return internal_memmove(dst, src, size); \
+ COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \
+ if (common_flags()->intercept_intrin) { \
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
+ } \
+ return REAL(memmove)(dst, src, size); \
+ }
+#endif
+
+#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL
+#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \
+ { \
+ if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \
+ return internal_memmove(dst, src, size); \
+ } \
+ COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \
+ if (common_flags()->intercept_intrin) { \
+ COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \
+ COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \
+ } \
+ return REAL(memcpy)(dst, src, size); \
+ }
+#endif
+
+#if SANITIZER_INTERCEPT_MEMSET
+INTERCEPTOR(void *, memset, void *dst, int v, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size);
+}
+
+#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset)
+#else
+#define INIT_MEMSET
+#endif
+
+#if SANITIZER_INTERCEPT_MEMMOVE
+INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+}
+
+#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove)
+#else
+#define INIT_MEMMOVE
+#endif
+
+#if SANITIZER_INTERCEPT_MEMCPY
+INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) {
+ // On OS X, calling internal_memcpy here will cause memory corruptions,
+ // because memcpy and memmove are actually aliases of the same
+ // implementation. We need to use internal_memmove here.
+ // N.B.: If we switch this to internal_ we'll have to use internal_memmove
+ // due to memcpy being an alias of memmove on OS X.
+ void *ctx;
+#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE
+ COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size);
+#else
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size);
+#endif
+}
+
+#define INIT_MEMCPY \
+ do { \
+ if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \
+ COMMON_INTERCEPT_FUNCTION(memcpy); \
+ } else { \
+ ASSIGN_REAL(memcpy, memmove); \
+ } \
+ CHECK(REAL(memcpy)); \
+ } while (false)
+
+#else
+#define INIT_MEMCPY
+#endif
+
+#if SANITIZER_INTERCEPT_AEABI_MEM
+INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size);
+}
+
+// Note the argument order.
+INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+
+#define INIT_AEABI_MEM \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \
+ COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8);
+#else
+#define INIT_AEABI_MEM
+#endif // SANITIZER_INTERCEPT_AEABI_MEM
+
+#if SANITIZER_INTERCEPT___BZERO
+INTERCEPTOR(void *, __bzero, void *block, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero);
+#else
+#define INIT___BZERO
+#endif // SANITIZER_INTERCEPT___BZERO
+
+#if SANITIZER_INTERCEPT_BZERO
+INTERCEPTOR(void *, bzero, void *block, uptr size) {
+ void *ctx;
+ COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size);
+}
+#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero);
+#else
+#define INIT_BZERO
+#endif // SANITIZER_INTERCEPT_BZERO
+
+namespace __sanitizer {
+// This does not need to be called if InitializeCommonInterceptors() is called.
+void InitializeMemintrinsicInterceptors() {
+ INIT_MEMSET;
+ INIT_MEMMOVE;
+ INIT_MEMCPY;
+ INIT_AEABI_MEM;
+ INIT___BZERO;
+ INIT_BZERO;
+}
+} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
index 958f071e7b5f7..557207fe62ac6 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
@@ -32,7 +32,9 @@ INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc)
INTERFACE_FUNCTION(__sanitizer_symbolize_global)
INTERFACE_FUNCTION(__sanitizer_symbolize_pc)
// Allocator interface.
+INTERFACE_FUNCTION(__sanitizer_get_allocated_begin)
INTERFACE_FUNCTION(__sanitizer_get_allocated_size)
+INTERFACE_FUNCTION(__sanitizer_get_allocated_size_fast)
INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes)
INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size)
INTERFACE_FUNCTION(__sanitizer_get_free_bytes)
@@ -44,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator)
INTERFACE_FUNCTION(__sanitizer_print_memory_profile)
INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook)
INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook)
+// Memintrinsic functions.
+INTERFACE_FUNCTION(__sanitizer_internal_memcpy)
+INTERFACE_FUNCTION(__sanitizer_internal_memmove)
+INTERFACE_FUNCTION(__sanitizer_internal_memset)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
index 4c565c5a361b0..85b89866f8fcd 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp
@@ -67,6 +67,8 @@ void *BackgroundThread(void *arg) {
} else if (soft_rss_limit_mb >= current_rss_mb &&
reached_soft_rss_limit) {
reached_soft_rss_limit = false;
+ Report("%s: soft rss limit unexhausted (%zdMb vs %zdMb)\n",
+ SanitizerToolName, soft_rss_limit_mb, current_rss_mb);
SetRssLimitExceeded(false);
}
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
index 9d7518ac9476c..c10943b3e4879 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc
@@ -417,14 +417,14 @@ PRE_SYSCALL(capget)(void *header, void *dataptr) {
POST_SYSCALL(capget)(long res, void *header, void *dataptr) {
if (res >= 0)
if (dataptr)
- POST_WRITE(dataptr, __user_cap_data_struct_sz);
+ POST_WRITE(dataptr, __user_cap_data_struct_sz(header));
}
PRE_SYSCALL(capset)(void *header, const void *data) {
if (header)
PRE_READ(header, __user_cap_header_struct_sz);
if (data)
- PRE_READ(data, __user_cap_data_struct_sz);
+ PRE_READ(data, __user_cap_data_struct_sz(header));
}
POST_SYSCALL(capset)(long res, void *header, const void *data) {}
@@ -1374,9 +1374,8 @@ PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) {
}
POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) {
- if (res >= 0) {
- if (ctx)
- POST_WRITE(ctx, sizeof(*ctx));
+ if (res >= 0 && ctx) {
+ POST_WRITE(ctx, sizeof(*ctx));
// (*ctx) is actually a pointer to a kernel mapped page, and there are
// people out there who are crazy enough to peek into that page's 32-byte
// header.
@@ -2136,7 +2135,7 @@ PRE_SYSCALL(epoll_pwait2)
const sanitizer_kernel_timespec *timeout, const kernel_sigset_t *sigmask,
long sigsetsize) {
if (timeout)
- PRE_READ(timeout, sizeof(timeout));
+ PRE_READ(timeout, sizeof(*timeout));
if (sigmask)
PRE_READ(sigmask, sigsetsize);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
index 956b48e0b434b..ce4326967180d 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp
@@ -282,7 +282,14 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, const uptr* beg,
// Weak definition for code instrumented with -fsanitize-coverage=stack-depth
// and later linked with code containing a strong definition.
// E.g., -fsanitize=fuzzer-no-link
+// FIXME: Update Apple deployment target so that thread_local is always
+// supported, and remove the #if.
+// FIXME: Figure out how this should work on Windows, exported thread_local
+// symbols are not supported:
+// "data with thread storage duration may not have dll interface"
+#if !SANITIZER_APPLE && !SANITIZER_WINDOWS
SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
-SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack;
+thread_local uptr __sancov_lowest_stack;
+#endif
#endif // !SANITIZER_FUCHSIA
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
index 810c1e452f611..9459c6b00accc 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h
@@ -15,6 +15,7 @@
#ifndef SANITIZER_FILE_H
#define SANITIZER_FILE_H
+#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
#include "sanitizer_mutex.h"
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp
index 9e274268bf2a6..c620da7f220af 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp
@@ -13,9 +13,9 @@
#include "sanitizer_flag_parser.h"
#include "sanitizer_common.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_flags.h"
#include "sanitizer_flag_parser.h"
+#include "sanitizer_flags.h"
+#include "sanitizer_libc.h"
namespace __sanitizer {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
index 3ccc6a6fa5377..ae49294dde95c 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h
@@ -13,9 +13,9 @@
#ifndef SANITIZER_FLAG_REGISTRY_H
#define SANITIZER_FLAG_REGISTRY_H
+#include "sanitizer_common.h"
#include "sanitizer_internal_defs.h"
#include "sanitizer_libc.h"
-#include "sanitizer_common.h"
namespace __sanitizer {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h
index 05fb554d20c14..8bb8304910c73 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h
@@ -21,12 +21,6 @@
namespace __sanitizer {
-// Call these callbacks on mmap/munmap.
-struct NoOpMapUnmapCallback {
- void OnMap(uptr p, uptr size) const {}
- void OnUnmap(uptr p, uptr size) const {}
-};
-
// Maps integers in rage [0, kSize) to values.
template
@@ -62,8 +56,7 @@ class FlatMap {
// Each value is initially zero and can be set to something else only once.
// Setting and getting values from multiple threads is safe w/o extra locking.
template
+ typename AddressSpaceViewTy = LocalAddressSpaceView>
class TwoLevelMap {
static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance.");
@@ -79,7 +72,6 @@ class TwoLevelMap {
T *p = Get(i);
if (!p)
continue;
- MapUnmapCallback().OnUnmap(reinterpret_cast(p), MmapSize());
UnmapOrDie(p, kSize2);
}
Init();
@@ -149,7 +141,6 @@ class TwoLevelMap {
T *res = Get(idx);
if (!res) {
res = reinterpret_cast(MmapOrDie(MmapSize(), "TwoLevelMap"));
- MapUnmapCallback().OnMap(reinterpret_cast(res), kSize2);
atomic_store(&map1_[idx], reinterpret_cast(res),
memory_order_release);
}
@@ -164,10 +155,8 @@ template
using FlatByteMap = FlatMap;
template
-using TwoLevelByteMap =
- TwoLevelMap;
+ typename AddressSpaceViewTy = LocalAddressSpaceView>
+using TwoLevelByteMap = TwoLevelMap;
} // namespace __sanitizer
#endif
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
index a92e84cb8ecf7..1e25265c00a20 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
@@ -285,6 +285,12 @@ bool MprotectReadOnly(uptr addr, uptr size) {
ZX_OK;
}
+bool MprotectReadWrite(uptr addr, uptr size) {
+ return _zx_vmar_protect(_zx_vmar_root_self(),
+ ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, addr,
+ size) == ZX_OK;
+}
+
void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment,
const char *mem_type) {
CHECK_GE(size, GetPageSize());
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
index 9683b97ab91d6..16b2a10d8b069 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc
@@ -1267,8 +1267,6 @@ static void ioctl_table_fill() {
_(TIOCGFLAGS, WRITE, sizeof(int));
_(TIOCSFLAGS, READ, sizeof(int));
_(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz);
- _(TIOCRCVFRAME, READ, sizeof(uptr));
- _(TIOCXMTFRAME, READ, sizeof(uptr));
_(TIOCPTMGET, WRITE, struct_ptmget_sz);
_(TIOCGRANTPT, NONE, 0);
_(TIOCPTSNAME, WRITE, struct_ptmget_sz);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
index b9090f47beb7b..63ee6e39f664b 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -13,6 +13,7 @@
#define SANITIZER_DEFS_H
#include "sanitizer_platform.h"
+#include "sanitizer_redefine_builtins.h"
#ifndef SANITIZER_DEBUG
# define SANITIZER_DEBUG 0
@@ -37,15 +38,6 @@
# define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak))
#endif
-// TLS is handled differently on different platforms
-#if SANITIZER_LINUX || SANITIZER_NETBSD || \
- SANITIZER_FREEBSD
-# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \
- __attribute__((tls_model("initial-exec"))) thread_local
-#else
-# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE
-#endif
-
//--------------------------- WEAK FUNCTIONS ---------------------------------//
// When working with weak functions, to simplify the code and make it more
// portable, when possible define a default implementation using this macro:
@@ -226,7 +218,7 @@ typedef u64 tid_t;
# define WARN_UNUSED_RESULT
#else // _MSC_VER
# define ALWAYS_INLINE inline __attribute__((always_inline))
-# define ALIAS(x) __attribute__((alias(x)))
+# define ALIAS(x) __attribute__((alias(SANITIZER_STRINGIFY(x))))
// Please only use the ALIGNED macro before the type.
// Using ALIGNED after the variable declaration is not portable!
# define ALIGNED(x) __attribute__((aligned(x)))
@@ -267,6 +259,12 @@ typedef u64 tid_t;
# define FALLTHROUGH
#endif
+#if __has_attribute(uninitialized)
+# define UNINITIALIZED __attribute__((uninitialized))
+#else
+# define UNINITIALIZED
+#endif
+
// Unaligned versions of basic types.
typedef ALIGNED(1) u16 uu16;
typedef ALIGNED(1) u32 uu32;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp
index d3076f0da4891..4a6fa5e8dbacb 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp
@@ -10,6 +10,9 @@
// run-time libraries. See sanitizer_libc.h for details.
//===----------------------------------------------------------------------===//
+// Do not redefine builtins; this file is defining the builtin replacements.
+#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+
#include "sanitizer_allocator_internal.h"
#include "sanitizer_common.h"
#include "sanitizer_libc.h"
@@ -46,7 +49,10 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) {
return 0;
}
-void *internal_memcpy(void *dest, const void *src, uptr n) {
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest,
+ const void *src,
+ uptr n) {
char *d = (char*)dest;
const char *s = (const char *)src;
for (uptr i = 0; i < n; ++i)
@@ -54,7 +60,8 @@ void *internal_memcpy(void *dest, const void *src, uptr n) {
return dest;
}
-void *internal_memmove(void *dest, const void *src, uptr n) {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
+ void *dest, const void *src, uptr n) {
char *d = (char*)dest;
const char *s = (const char *)src;
sptr i, signed_n = (sptr)n;
@@ -72,7 +79,8 @@ void *internal_memmove(void *dest, const void *src, uptr n) {
return dest;
}
-void *internal_memset(void* s, int c, uptr n) {
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c,
+ uptr n) {
// Optimize for the most performance-critical case:
if ((reinterpret_cast(s) % 16) == 0 && (n % 16) == 0) {
u64 *p = reinterpret_cast(s);
@@ -95,6 +103,7 @@ void *internal_memset(void* s, int c, uptr n) {
}
return s;
}
+} // extern "C"
uptr internal_strcspn(const char *s, const char *reject) {
uptr i;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
index 39a212665d0ae..e881db2079086 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h
@@ -24,15 +24,33 @@ namespace __sanitizer {
// internal_X() is a custom implementation of X() for use in RTL.
+extern "C" {
+// These are used as builtin replacements; see sanitizer_redefine_builtins.h.
+// In normal runtime code, use the __sanitizer::internal_X() aliases instead.
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest,
+ const void *src,
+ uptr n);
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove(
+ void *dest, const void *src, uptr n);
+SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c,
+ uptr n);
+} // extern "C"
+
// String functions
s64 internal_atoll(const char *nptr);
void *internal_memchr(const void *s, int c, uptr n);
void *internal_memrchr(const void *s, int c, uptr n);
int internal_memcmp(const void* s1, const void* s2, uptr n);
-void *internal_memcpy(void *dest, const void *src, uptr n);
-void *internal_memmove(void *dest, const void *src, uptr n);
+ALWAYS_INLINE void *internal_memcpy(void *dest, const void *src, uptr n) {
+ return __sanitizer_internal_memcpy(dest, src, n);
+}
+ALWAYS_INLINE void *internal_memmove(void *dest, const void *src, uptr n) {
+ return __sanitizer_internal_memmove(dest, src, n);
+}
// Should not be used in performance-critical places.
-void *internal_memset(void *s, int c, uptr n);
+ALWAYS_INLINE void *internal_memset(void *s, int c, uptr n) {
+ return __sanitizer_internal_memset(s, c, n);
+}
char* internal_strchr(const char *s, int c);
char *internal_strchrnul(const char *s, int c);
int internal_strcmp(const char *s1, const char *s2);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
index dada262b02b16..90d5858a72037 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp
@@ -168,11 +168,11 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
namespace __sanitizer {
-void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *old) {
- CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, old));
+void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) {
+ CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset));
}
-ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
+void BlockSignals(__sanitizer_sigset_t *oldset) {
__sanitizer_sigset_t set;
internal_sigfillset(&set);
# if SANITIZER_LINUX && !SANITIZER_ANDROID
@@ -187,7 +187,11 @@ ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
// hang.
internal_sigdelset(&set, 31);
# endif
- SetSigProcMask(&set, &saved_);
+ SetSigProcMask(&set, oldset);
+}
+
+ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) {
+ BlockSignals(&saved_);
if (copy)
internal_memcpy(copy, &saved_, sizeof(saved_));
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
index 87e4959db6fba..d5655eab12a68 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h
@@ -51,6 +51,7 @@ uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set,
__sanitizer_sigset_t *oldset);
void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset);
+void BlockSignals(__sanitizer_sigset_t *oldset = nullptr);
struct ScopedBlockSignals {
explicit ScopedBlockSignals(__sanitizer_sigset_t *copy);
~ScopedBlockSignals();
@@ -152,6 +153,9 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
"rdhwr %0,$29\n" \
".set pop\n" : "=r"(__v)); \
__v; })
+#elif defined (__riscv)
+# define __get_tls() \
+ ({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; })
#elif defined(__i386__)
# define __get_tls() \
({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; })
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
index 8777e4c4556e7..1baab2bef68cf 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp
@@ -148,7 +148,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top,
pthread_attr_t attr;
pthread_attr_init(&attr);
CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0);
- my_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
+ internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize);
pthread_attr_destroy(&attr);
#endif // SANITIZER_SOLARIS
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 23c4c6619de82..24e3d1112520e 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -12,81 +12,81 @@
#include "sanitizer_platform.h"
#if SANITIZER_APPLE
-#include "sanitizer_mac.h"
-#include "interception/interception.h"
+# include "interception/interception.h"
+# include "sanitizer_mac.h"
// Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so
// the clients will most certainly use 64-bit ones as well.
-#ifndef _DARWIN_USE_64_BIT_INODE
-#define _DARWIN_USE_64_BIT_INODE 1
-#endif
-#include
-
-#include "sanitizer_common.h"
-#include "sanitizer_file.h"
-#include "sanitizer_flags.h"
-#include "sanitizer_interface_internal.h"
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_libc.h"
-#include "sanitizer_platform_limits_posix.h"
-#include "sanitizer_procmaps.h"
-#include "sanitizer_ptrauth.h"
-
-#if !SANITIZER_IOS
-#include // for _NSGetEnviron
-#else
+# ifndef _DARWIN_USE_64_BIT_INODE
+# define _DARWIN_USE_64_BIT_INODE 1
+# endif
+# include
+
+# include "sanitizer_common.h"
+# include "sanitizer_file.h"
+# include "sanitizer_flags.h"
+# include "sanitizer_interface_internal.h"
+# include "sanitizer_internal_defs.h"
+# include "sanitizer_libc.h"
+# include "sanitizer_platform_limits_posix.h"
+# include "sanitizer_procmaps.h"
+# include "sanitizer_ptrauth.h"
+
+# if !SANITIZER_IOS
+# include // for _NSGetEnviron
+# else
extern char **environ;
-#endif
+# endif
-#if defined(__has_include) && __has_include()
-#define SANITIZER_OS_TRACE 1
-#include
-#else
-#define SANITIZER_OS_TRACE 0
-#endif
+# if defined(__has_include) && __has_include()
+# define SANITIZER_OS_TRACE 1
+# include
+# else
+# define SANITIZER_OS_TRACE 0
+# endif
// import new crash reporting api
-#if defined(__has_include) && __has_include()
-#define HAVE_CRASHREPORTERCLIENT_H 1
-#include
-#else
-#define HAVE_CRASHREPORTERCLIENT_H 0
-#endif
+# if defined(__has_include) && __has_include()
+# define HAVE_CRASHREPORTERCLIENT_H 1
+# include
+# else
+# define HAVE_CRASHREPORTERCLIENT_H 0
+# endif
-#if !SANITIZER_IOS
-#include // for _NSGetArgv and _NSGetEnviron
-#else
+# if !SANITIZER_IOS
+# include // for _NSGetArgv and _NSGetEnviron
+# else
extern "C" {
- extern char ***_NSGetArgv(void);
+extern char ***_NSGetArgv(void);
}
-#endif
+# endif
-#include
-#include // for dladdr()
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
-#include
+# include
+# include // for dladdr()
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
+# include
// From , but we don't have that file on iOS.
extern "C" {
@@ -989,7 +989,7 @@ static void VerifyInterceptorsWorking() {
// "wrap_puts" within our own dylib.
Dl_info info_puts, info_runtime;
RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts));
- RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info_runtime));
+ RAW_CHECK(dladdr((void *)&VerifyInterceptorsWorking, &info_runtime));
if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) {
Report(
"ERROR: Interceptors are not working. This may be because %s is "
@@ -1039,7 +1039,7 @@ static void StripEnv() {
return;
Dl_info info;
- RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info));
+ RAW_CHECK(dladdr((void *)&StripEnv, &info));
const char *dylib_name = StripModuleName(info.dli_fname);
bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name);
if (!lib_is_in_env)
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h
new file mode 100644
index 0000000000000..4e58c02df8351
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h
@@ -0,0 +1,38 @@
+//===-- sanitizer_mallinfo.h ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of Sanitizer common code.
+//
+// Definition for mallinfo on different platforms.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_MALLINFO_H
+#define SANITIZER_MALLINFO_H
+
+#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
+
+namespace __sanitizer {
+
+#if SANITIZER_ANDROID
+
+struct __sanitizer_struct_mallinfo {
+ uptr v[10];
+};
+
+#elif SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_FUCHSIA
+
+struct __sanitizer_struct_mallinfo {
+ int v[10];
+};
+
+#endif
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_MALLINFO_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index 36acd5f2d3395..0df653463f436 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -359,7 +359,8 @@
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \
(SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC
-#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP \
+ (SI_LINUX_NOT_ANDROID || SI_FREEBSD)
#define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX
#define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \
(SI_POSIX && !SI_NETBSD)
@@ -379,6 +380,8 @@
(SI_LINUX_NOT_ANDROID || SI_SOLARIS)
#define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \
(SI_LINUX_NOT_ANDROID && !SI_NETBSD)
+#define SANITIZER_INTERCEPT_TRYJOIN SI_GLIBC
+#define SANITIZER_INTERCEPT_TIMEDJOIN SI_GLIBC
#define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD
#define SANITIZER_INTERCEPT_TMPNAM SI_POSIX
#define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC || SI_SOLARIS)
@@ -504,6 +507,7 @@
#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC)
#define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD)
#define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID
+#define SANITIZER_INTERCEPT_WCSLEN 1
#define SANITIZER_INTERCEPT_WCSCAT SI_POSIX
#define SANITIZER_INTERCEPT_WCSDUP SI_POSIX
#define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA)
@@ -581,15 +585,14 @@
#define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD
#define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID)
#define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
-#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID)
-#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID)
#define SANITIZER_INTERCEPT_GETRANDOM \
((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
#define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
#define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
#define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
-#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD
+#define SANITIZER_INTERCEPT_GETENTROPY \
+ ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD)
#define SANITIZER_INTERCEPT_QSORT \
(SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID)
#define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC
@@ -604,6 +607,8 @@
#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD
#define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD
#define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD
+#define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC
+#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD
// This macro gives a way for downstream users to override the above
// interceptor macros irrespective of the platform they are on. They have
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
index 37e72cd5d45ea..38f968d533b14 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
@@ -17,6 +17,7 @@
#include
#include
+#include
#include
#include
#include
@@ -103,6 +104,7 @@ void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) {
return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr;
}
+unsigned struct_cpuset_sz = sizeof(cpuset_t);
unsigned struct_cap_rights_sz = sizeof(cap_rights_t);
unsigned struct_utsname_sz = sizeof(struct utsname);
unsigned struct_stat_sz = sizeof(struct stat);
@@ -173,6 +175,12 @@ uptr __sanitizer_in_addr_sz(int af) {
return 0;
}
+// For FreeBSD the actual size of a directory entry is not always in d_reclen.
+// Use the appropriate macro to get the correct size for all cases (e.g. NFS).
+u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp) {
+ return _GENERIC_DIRSIZ(dp);
+}
+
unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr);
int glob_nomatch = GLOB_NOMATCH;
int glob_altdirfunc = GLOB_ALTDIRFUNC;
@@ -558,4 +566,5 @@ COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE);
CHECK_TYPE_SIZE(sem_t);
COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t));
+COMPILER_CHECK(sizeof(__sanitizer_cpuset_t) >= sizeof(cpuset_t));
#endif // SANITIZER_FREEBSD
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
index daef1177a2dba..b119f059007d8 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
@@ -249,9 +249,15 @@ struct __sanitizer_dirent {
unsigned int d_fileno;
# endif
unsigned short d_reclen;
- // more fields that we don't care about
+ u8 d_type;
+ u8 d_pad0;
+ u16 d_namlen;
+ u16 d_pad1;
+ char d_name[256];
};
+u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp);
+
// 'clock_t' is 32 bits wide on x64 FreeBSD
typedef int __sanitizer_clock_t;
typedef int __sanitizer_clockid_t;
@@ -709,6 +715,17 @@ extern unsigned struct_cap_rights_sz;
extern unsigned struct_fstab_sz;
extern unsigned struct_StringList_sz;
+
+struct __sanitizer_cpuset {
+#if __FreeBSD_version >= 1400090
+ long __bits[(1024 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
+#else
+ long __bits[(256 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)];
+#endif
+};
+
+typedef struct __sanitizer_cpuset __sanitizer_cpuset_t;
+extern unsigned struct_cpuset_sz;
} // namespace __sanitizer
# define CHECK_TYPE_SIZE(TYPE) \
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
index 648e502b904a5..c40877ba48d07 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
@@ -2342,8 +2342,6 @@ unsigned IOCTL_TIOCDRAIN = TIOCDRAIN;
unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS;
unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS;
unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP;
-unsigned IOCTL_TIOCRCVFRAME = TIOCRCVFRAME;
-unsigned IOCTL_TIOCXMTFRAME = TIOCXMTFRAME;
unsigned IOCTL_TIOCPTMGET = TIOCPTMGET;
unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT;
unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
index dc6eb59b2800e..4c697b4d107dd 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h
@@ -2195,8 +2195,6 @@ extern unsigned IOCTL_TIOCDRAIN;
extern unsigned IOCTL_TIOCGFLAGS;
extern unsigned IOCTL_TIOCSFLAGS;
extern unsigned IOCTL_TIOCDCDTIMESTAMP;
-extern unsigned IOCTL_TIOCRCVFRAME;
-extern unsigned IOCTL_TIOCXMTFRAME;
extern unsigned IOCTL_TIOCPTMGET;
extern unsigned IOCTL_TIOCGRANTPT;
extern unsigned IOCTL_TIOCPTSNAME;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
index 8db81c51a5a49..72d76bb581531 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp
@@ -18,6 +18,7 @@
// depends on _FILE_OFFSET_BITS setting.
// To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below.
#undef _FILE_OFFSET_BITS
+#undef _TIME_BITS
#endif
// Must go after undef _FILE_OFFSET_BITS.
@@ -176,10 +177,6 @@ typedef struct user_fpregs elf_fpregset_t;
# include "sanitizer_platform_interceptors.h"
# include "sanitizer_platform_limits_posix.h"
-#if SANITIZER_INTERCEPT_CRYPT_R
-#include
-#endif
-
namespace __sanitizer {
unsigned struct_utsname_sz = sizeof(struct utsname);
unsigned struct_stat_sz = sizeof(struct stat);
@@ -247,7 +244,23 @@ namespace __sanitizer {
unsigned struct_sysinfo_sz = sizeof(struct sysinfo);
unsigned __user_cap_header_struct_sz =
sizeof(struct __user_cap_header_struct);
- unsigned __user_cap_data_struct_sz = sizeof(struct __user_cap_data_struct);
+ unsigned __user_cap_data_struct_sz(void *hdrp) {
+ int u32s = 0;
+ if (hdrp) {
+ switch (((struct __user_cap_header_struct *)hdrp)->version) {
+ case _LINUX_CAPABILITY_VERSION_1:
+ u32s = _LINUX_CAPABILITY_U32S_1;
+ break;
+ case _LINUX_CAPABILITY_VERSION_2:
+ u32s = _LINUX_CAPABILITY_U32S_2;
+ break;
+ case _LINUX_CAPABILITY_VERSION_3:
+ u32s = _LINUX_CAPABILITY_U32S_3;
+ break;
+ }
+ }
+ return sizeof(struct __user_cap_data_struct) * u32s;
+ }
unsigned struct_new_utsname_sz = sizeof(struct new_utsname);
unsigned struct_old_utsname_sz = sizeof(struct old_utsname);
unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname);
@@ -283,10 +296,6 @@ namespace __sanitizer {
unsigned struct_statvfs64_sz = sizeof(struct statvfs64);
#endif // SANITIZER_GLIBC
-#if SANITIZER_INTERCEPT_CRYPT_R
- unsigned struct_crypt_data_sz = sizeof(struct crypt_data);
-#endif
-
#if SANITIZER_LINUX && !SANITIZER_ANDROID
unsigned struct_timex_sz = sizeof(struct timex);
unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
index 11d39b80717f0..08d1a685208be 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h
@@ -18,6 +18,7 @@
#include "sanitizer_internal_defs.h"
#include "sanitizer_platform.h"
+#include "sanitizer_mallinfo.h"
#if SANITIZER_APPLE
#include
@@ -139,7 +140,7 @@ struct __sanitizer_perf_event_attr {
extern unsigned struct_epoll_event_sz;
extern unsigned struct_sysinfo_sz;
extern unsigned __user_cap_header_struct_sz;
-extern unsigned __user_cap_data_struct_sz;
+extern unsigned __user_cap_data_struct_sz(void *hdrp);
extern unsigned struct_new_utsname_sz;
extern unsigned struct_old_utsname_sz;
extern unsigned struct_oldold_utsname_sz;
@@ -208,17 +209,7 @@ struct __sanitizer_sem_t {
};
#endif // SANITIZER_LINUX
-#if SANITIZER_ANDROID
-struct __sanitizer_struct_mallinfo {
- uptr v[10];
-};
-#endif
-
#if SANITIZER_LINUX && !SANITIZER_ANDROID
-struct __sanitizer_struct_mallinfo {
- int v[10];
-};
-
extern unsigned struct_ustat_sz;
extern unsigned struct_rlimit64_sz;
extern unsigned struct_statvfs64_sz;
@@ -322,7 +313,6 @@ extern unsigned struct_msqid_ds_sz;
extern unsigned struct_mq_attr_sz;
extern unsigned struct_timex_sz;
extern unsigned struct_statvfs_sz;
-extern unsigned struct_crypt_data_sz;
#endif // SANITIZER_LINUX && !SANITIZER_ANDROID
struct __sanitizer_iovec {
@@ -596,8 +586,13 @@ typedef unsigned long __sanitizer_sigset_t;
#endif
struct __sanitizer_siginfo_pad {
+#if SANITIZER_X32
+ // x32 siginfo_t is aligned to 8 bytes.
+ u64 pad[128 / sizeof(u64)];
+#else
// Require uptr, because siginfo_t is always pointer-size aligned on Linux.
uptr pad[128 / sizeof(uptr)];
+#endif
};
#if SANITIZER_LINUX
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
index 00e49ecbf327e..ae8f8fe07c588 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp
@@ -57,11 +57,9 @@ void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) {
void UnmapOrDie(void *addr, uptr size) {
if (!addr || !size) return;
uptr res = internal_munmap(addr, size);
- if (UNLIKELY(internal_iserror(res))) {
- Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n",
- SanitizerToolName, size, size, addr);
- CHECK("unable to unmap" && 0);
- }
+ int reserrno;
+ if (UNLIKELY(internal_iserror(res, &reserrno)))
+ ReportMunmapFailureAndDie(addr, size, reserrno);
DecreaseTotalMmap(size);
}
@@ -160,6 +158,14 @@ bool MprotectReadOnly(uptr addr, uptr size) {
#endif
}
+bool MprotectReadWrite(uptr addr, uptr size) {
+#if SANITIZER_EMSCRIPTEN
+ return true;
+#else
+ return 0 == internal_mprotect((void *)addr, size, PROT_READ | PROT_WRITE);
+#endif
+}
+
#if !SANITIZER_APPLE
void MprotectMallocZones(void *addr, int prot) {}
#endif
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
index f91e26e74b87c..c5811dffea94b 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h
@@ -90,7 +90,7 @@ int real_pthread_join(void *th, void **ret);
} \
} // namespace __sanitizer
-int my_pthread_attr_getstack(void *attr, void **addr, uptr *size);
+int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size);
// A routine named real_sigaction() must be implemented by each sanitizer in
// order for internal_sigaction() to bypass interceptors.
@@ -120,6 +120,9 @@ int GetNamedMappingFd(const char *name, uptr size, int *flags);
// alive at least as long as the mapping exists.
void DecorateMapping(uptr addr, uptr size, const char *name);
+# if !SANITIZER_FREEBSD
+# define __sanitizer_dirsiz(dp) ((dp)->d_reclen)
+# endif
} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
index b545453641fca..2c34bcbc8bf0e 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -393,7 +393,7 @@ SANITIZER_WEAK_ATTRIBUTE int
real_pthread_attr_getstack(void *attr, void **addr, size_t *size);
} // extern "C"
-int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) {
+int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size) {
#if !SANITIZER_GO && !SANITIZER_APPLE
if (&real_pthread_attr_getstack)
return real_pthread_attr_getstack((pthread_attr_t *)attr, addr,
@@ -407,7 +407,7 @@ void AdjustStackSize(void *attr_) {
pthread_attr_t *attr = (pthread_attr_t *)attr_;
uptr stackaddr = 0;
uptr stacksize = 0;
- my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize);
+ internal_pthread_attr_getstack(attr, (void **)&stackaddr, &stacksize);
// GLibC will return (0 - stacksize) as the stack address in the case when
// stacksize is set, but stackaddr is not.
bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
index 19bad158387c5..bf3c2c28e32e3 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h
@@ -65,6 +65,8 @@ class MemoryMappedSegment {
MemoryMappedSegmentData *data_;
};
+struct ImageHeader;
+
class MemoryMappingLayoutBase {
public:
virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); }
@@ -75,10 +77,22 @@ class MemoryMappingLayoutBase {
~MemoryMappingLayoutBase() {}
};
-class MemoryMappingLayout final : public MemoryMappingLayoutBase {
+class MemoryMappingLayout : public MemoryMappingLayoutBase {
public:
explicit MemoryMappingLayout(bool cache_enabled);
+
+// This destructor cannot be virtual, as it would cause an operator new() linking
+// failures in hwasan test cases. However non-virtual destructors emit warnings
+// in macOS build, hence disabling those
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnon-virtual-dtor"
+#endif
~MemoryMappingLayout();
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
virtual bool Next(MemoryMappedSegment *segment) override;
virtual bool Error() const override;
virtual void Reset() override;
@@ -90,10 +104,14 @@ class MemoryMappingLayout final : public MemoryMappingLayoutBase {
// Adds all mapped objects into a vector.
void DumpListOfModules(InternalMmapVectorNoCtor *modules);
+ protected:
+#if SANITIZER_APPLE
+ virtual const ImageHeader *CurrentImageHeader();
+#endif
+ MemoryMappingLayoutData data_;
+
private:
void LoadFromCache();
-
- MemoryMappingLayoutData data_;
};
// Returns code range for the specified module.
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
index 4b0e678197614..b44e016a0e5bc 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp
@@ -250,7 +250,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment,
MemoryMappedSegmentData *seg_data,
MemoryMappingLayoutData *layout_data) {
const char *lc = layout_data->current_load_cmd_addr;
+
layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize;
+ layout_data->current_load_cmd_count--;
if (((const load_command *)lc)->cmd == kLCSegment) {
const SegmentCommand* sc = (const SegmentCommand *)lc;
uptr base_virt_addr, addr_mask;
@@ -358,11 +360,16 @@ static bool IsModuleInstrumented(const load_command *first_lc) {
return false;
}
+const ImageHeader *MemoryMappingLayout::CurrentImageHeader() {
+ const mach_header *hdr = (data_.current_image == kDyldImageIdx)
+ ? get_dyld_hdr()
+ : _dyld_get_image_header(data_.current_image);
+ return (const ImageHeader *)hdr;
+}
+
bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
for (; data_.current_image >= kDyldImageIdx; data_.current_image--) {
- const mach_header *hdr = (data_.current_image == kDyldImageIdx)
- ? get_dyld_hdr()
- : _dyld_get_image_header(data_.current_image);
+ const mach_header *hdr = (const mach_header *)CurrentImageHeader();
if (!hdr) continue;
if (data_.current_load_cmd_count < 0) {
// Set up for this image;
@@ -392,7 +399,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
(const load_command *)data_.current_load_cmd_addr);
}
- for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) {
+ while (data_.current_load_cmd_count > 0) {
switch (data_.current_magic) {
// data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64.
#ifdef MH_MAGIC_64
@@ -413,6 +420,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) {
}
// If we get here, no more load_cmd's in this image talk about
// segments. Go on to the next image.
+ data_.current_load_cmd_count = -1; // This will trigger loading next image
}
return false;
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
index 4aa6054851666..460d96ea681b4 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h
@@ -68,10 +68,6 @@ struct QuarantineBatch {
COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13)); // 8Kb.
-// The callback interface is:
-// void Callback::Recycle(Node *ptr);
-// void *cb.Allocate(uptr size);
-// void cb.Deallocate(void *ptr);
template
class Quarantine {
public:
@@ -94,21 +90,20 @@ class Quarantine {
recycle_mutex_.Init();
}
- uptr GetSize() const { return atomic_load_relaxed(&max_size_); }
- uptr GetCacheSize() const {
- return atomic_load_relaxed(&max_cache_size_);
- }
+ uptr GetMaxSize() const { return atomic_load_relaxed(&max_size_); }
+ uptr GetMaxCacheSize() const { return atomic_load_relaxed(&max_cache_size_); }
void Put(Cache *c, Callback cb, Node *ptr, uptr size) {
- uptr cache_size = GetCacheSize();
- if (cache_size) {
+ uptr max_cache_size = GetMaxCacheSize();
+ if (max_cache_size && size <= GetMaxSize()) {
+ cb.PreQuarantine(ptr);
c->Enqueue(cb, ptr, size);
} else {
- // GetCacheSize() == 0 only when GetSize() == 0 (see Init).
- cb.Recycle(ptr);
+ // GetMaxCacheSize() == 0 only when GetMaxSize() == 0 (see Init).
+ cb.RecyclePassThrough(ptr);
}
// Check cache size anyway to accommodate for runtime cache_size change.
- if (c->Size() > cache_size)
+ if (c->Size() > max_cache_size)
Drain(c, cb);
}
@@ -117,7 +112,7 @@ class Quarantine {
SpinMutexLock l(&cache_mutex_);
cache_.Transfer(c);
}
- if (cache_.Size() > GetSize() && recycle_mutex_.TryLock())
+ if (cache_.Size() > GetMaxSize() && recycle_mutex_.TryLock())
Recycle(atomic_load_relaxed(&min_size_), cb);
}
@@ -133,7 +128,7 @@ class Quarantine {
void PrintStats() const {
// It assumes that the world is stopped, just as the allocator's PrintStats.
Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n",
- GetSize() >> 20, GetCacheSize() >> 10);
+ GetMaxSize() >> 20, GetMaxCacheSize() >> 10);
cache_.PrintStats();
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp
new file mode 100644
index 0000000000000..68d79f18ac8d3
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp
@@ -0,0 +1,62 @@
+//===-- sanitizer_range.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_range.h"
+
+#include "sanitizer_common/sanitizer_array_ref.h"
+
+namespace __sanitizer {
+
+void Intersect(ArrayRef a, ArrayRef b,
+ InternalMmapVectorNoCtor &output) {
+ output.clear();
+
+ struct Event {
+ uptr val;
+ s8 diff1;
+ s8 diff2;
+ };
+
+ InternalMmapVector events;
+ for (const Range &r : a) {
+ CHECK_LE(r.begin, r.end);
+ events.push_back({r.begin, 1, 0});
+ events.push_back({r.end, -1, 0});
+ }
+
+ for (const Range &r : b) {
+ CHECK_LE(r.begin, r.end);
+ events.push_back({r.begin, 0, 1});
+ events.push_back({r.end, 0, -1});
+ }
+
+ Sort(events.data(), events.size(),
+ [](const Event &lh, const Event &rh) { return lh.val < rh.val; });
+
+ uptr start = 0;
+ sptr state1 = 0;
+ sptr state2 = 0;
+ for (const auto &e : events) {
+ if (e.val != start) {
+ DCHECK_GE(state1, 0);
+ DCHECK_GE(state2, 0);
+ if (state1 && state2) {
+ if (!output.empty() && start == output.back().end)
+ output.back().end = e.val;
+ else
+ output.push_back({start, e.val});
+ }
+ start = e.val;
+ }
+
+ state1 += e.diff1;
+ state2 += e.diff2;
+ }
+}
+
+} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h
new file mode 100644
index 0000000000000..7c593e171ba21
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h
@@ -0,0 +1,40 @@
+//===-- sanitizer_range.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Contais Range and related utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_RANGE_H
+#define SANITIZER_RANGE_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_common/sanitizer_array_ref.h"
+
+namespace __sanitizer {
+
+struct Range {
+ uptr begin;
+ uptr end;
+};
+
+inline bool operator==(const Range &lhs, const Range &rhs) {
+ return lhs.begin == rhs.begin && lhs.end == rhs.end;
+}
+
+inline bool operator!=(const Range &lhs, const Range &rhs) {
+ return !(lhs == rhs);
+}
+
+// Calculates intersection of two sets of regions in O(N log N) time.
+void Intersect(ArrayRef a, ArrayRef b,
+ InternalMmapVectorNoCtor &output);
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_RANGE_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
new file mode 100644
index 0000000000000..9e4c2028dc430
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
@@ -0,0 +1,53 @@
+//===-- sanitizer_redefine_builtins.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Redefine builtin functions to use internal versions. This is needed where
+// compiler optimizations end up producing unwanted libcalls!
+//
+//===----------------------------------------------------------------------===//
+#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS
+#ifndef SANITIZER_REDEFINE_BUILTINS_H
+#define SANITIZER_REDEFINE_BUILTINS_H
+
+// The asm hack only works with GCC and Clang.
+// XXX Emscripten This does not work in Wasm.
+#if !defined(_WIN32) && !defined(__wasm__)
+
+asm("memcpy = __sanitizer_internal_memcpy");
+asm("memmove = __sanitizer_internal_memmove");
+asm("memset = __sanitizer_internal_memset");
+
+// The builtins should not be redefined in source files that make use of C++
+// standard libraries, in particular where C++STL headers with inline functions
+// are used. The redefinition in such cases would lead to ODR violations.
+//
+// Try to break the build in common cases where builtins shouldn't be redefined.
+namespace std {
+class Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file {
+ Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file(
+ const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
+ Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file& operator=(
+ const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete;
+};
+using array = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using atomic = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using function = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using shared_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using string = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using unique_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using unordered_map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file;
+} // namespace std
+
+#endif // !_WIN32 && !__wasm__
+
+#endif // SANITIZER_REDEFINE_BUILTINS_H
+#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc
index 475e577d9982e..94e4e2954a3b9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc
@@ -43,6 +43,7 @@ using namespace __sanitizer;
#if SANITIZER_INTERCEPT_BSD_SIGNAL
INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
+ SIGNAL_INTERCEPTOR_ENTER();
if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0;
SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler);
}
@@ -53,6 +54,7 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) {
#if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION
INTERCEPTOR(uptr, signal, int signum, uptr handler) {
+ SIGNAL_INTERCEPTOR_ENTER();
if (GetHandleSignalMode(signum) == kHandleSignalExclusive)
return (uptr) nullptr;
SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler);
@@ -61,6 +63,7 @@ INTERCEPTOR(uptr, signal, int signum, uptr handler) {
INTERCEPTOR(int, sigaction_symname, int signum,
const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) {
+ SIGNAL_INTERCEPTOR_ENTER();
if (GetHandleSignalMode(signum) == kHandleSignalExclusive) {
if (!oldact) return 0;
act = nullptr;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
index 23acbff79c664..75ff41ed52c24 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h
@@ -94,10 +94,10 @@ uptr StackTrace::GetPreviousInstructionPc(uptr pc) {
#elif defined(__sparc__) || defined(__mips__)
return pc - 8;
#elif SANITIZER_RISCV64
- // RV-64 has variable instruciton length...
+ // RV-64 has variable instruction length...
// C extentions gives us 2-byte instructoins
// RV-64 has 4-byte instructions
- // + RISCV architecture allows instructions up to 8 bytes
+ // + RISC-V architecture allows instructions up to 8 bytes
// It seems difficult to figure out the exact instruction length -
// pc - 2 seems like a safe option for the purposes of stack tracing
return pc - 2;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
index 2d0eccc1602ab..45c480d225c7f 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp
@@ -11,25 +11,47 @@
//===----------------------------------------------------------------------===//
#include "sanitizer_stacktrace_printer.h"
+
#include "sanitizer_file.h"
+#include "sanitizer_flags.h"
#include "sanitizer_fuchsia.h"
namespace __sanitizer {
-// sanitizer_symbolizer_markup.cpp implements these differently.
-#if !SANITIZER_SYMBOLIZER_MARKUP
-
-static const char *StripFunctionName(const char *function, const char *prefix) {
- if (!function) return nullptr;
- if (!prefix) return function;
- uptr prefix_len = internal_strlen(prefix);
- if (0 == internal_strncmp(function, prefix, prefix_len))
- return function + prefix_len;
+const char *StripFunctionName(const char *function) {
+ if (!common_flags()->demangle)
+ return function;
+ if (!function)
+ return nullptr;
+ auto try_strip = [function](const char *prefix) -> const char * {
+ const uptr prefix_len = internal_strlen(prefix);
+ if (!internal_strncmp(function, prefix, prefix_len))
+ return function + prefix_len;
+ return nullptr;
+ };
+ if (SANITIZER_APPLE) {
+ if (const char *s = try_strip("wrap_"))
+ return s;
+ } else if (SANITIZER_WINDOWS) {
+ if (const char *s = try_strip("__asan_wrap_"))
+ return s;
+ } else {
+ if (const char *s = try_strip("___interceptor_"))
+ return s;
+ if (const char *s = try_strip("__interceptor_"))
+ return s;
+ }
return function;
}
+// sanitizer_symbolizer_markup.cpp implements these differently.
+#if !SANITIZER_SYMBOLIZER_MARKUP
+
static const char *DemangleFunctionName(const char *function) {
- if (!function) return nullptr;
+ if (!common_flags()->demangle)
+ return function;
+ if (!function)
+ return nullptr;
// NetBSD uses indirection for old threading functions for historical reasons
// The mangled names are internal implementation detail and should not be
@@ -121,7 +143,7 @@ static const char kDefaultFormat[] = " #%n %p %F %L";
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
uptr address, const AddressInfo *info, bool vs_style,
- const char *strip_path_prefix, const char *strip_func_prefix) {
+ const char *strip_path_prefix) {
// info will be null in the case where symbolization is not needed for the
// given format. This ensures that the code below will get a hard failure
// rather than print incorrect information in case RenderNeedsSymbolization
@@ -157,8 +179,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer);
break;
case 'f':
- buffer->append("%s", DemangleFunctionName(StripFunctionName(
- info->function, strip_func_prefix)));
+ buffer->append("%s",
+ DemangleFunctionName(StripFunctionName(info->function)));
break;
case 'q':
buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown
@@ -178,8 +200,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
case 'F':
// Function name and offset, if file is unknown.
if (info->function) {
- buffer->append("in %s", DemangleFunctionName(StripFunctionName(
- info->function, strip_func_prefix)));
+ buffer->append("in %s",
+ DemangleFunctionName(StripFunctionName(info->function)));
if (!info->file && info->function_offset != AddressInfo::kUnknown)
buffer->append("+0x%zx", info->function_offset);
}
@@ -198,7 +220,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
RenderModuleLocation(buffer, info->module, info->module_offset,
info->module_arch, strip_path_prefix);
+#if !SANITIZER_APPLE
MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
+#endif
} else {
buffer->append("()");
}
@@ -211,7 +235,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
// Always strip the module name for %M.
RenderModuleLocation(buffer, StripModuleName(info->module),
info->module_offset, info->module_arch, "");
+#if !SANITIZER_APPLE
MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer);
+#endif
} else {
buffer->append("(%p)", (void *)address);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h
index 96119b2ee9e9f..bf2755a2e8f45 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h
@@ -17,6 +17,9 @@
namespace __sanitizer {
+// Strip interceptor prefixes from function name.
+const char *StripFunctionName(const char *function);
+
// Render the contents of "info" structure, which represents the contents of
// stack frame "frame_no" and appends it to the "buffer". "format" is a
// string with placeholders, which is copied to the output with
@@ -26,8 +29,7 @@ namespace __sanitizer {
// will be turned into
// " frame 10: function foo::bar() at my/file.cc:10"
// You may additionally pass "strip_path_prefix" to strip prefixes of paths to
-// source files and modules, and "strip_func_prefix" to strip prefixes of
-// function names.
+// source files and modules.
// Here's the full list of available placeholders:
// %% - represents a '%' character;
// %n - frame number (copy of frame_no);
@@ -48,8 +50,7 @@ namespace __sanitizer {
// %M - prints module basename and offset, if it is known, or PC.
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
uptr address, const AddressInfo *info, bool vs_style,
- const char *strip_path_prefix = "",
- const char *strip_func_prefix = "");
+ const char *strip_path_prefix = "");
bool RenderNeedsSymbolization(const char *format);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp
index 1e635a66978f3..a2000798a3907 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp
@@ -30,13 +30,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top,
// TODO(yln): add arg sanity check for stack_top/stack_bottom
CHECK_GE(max_depth, 2);
const uptr kPageSize = GetPageSizeCached();
-#if defined(__GNUC__)
- // __builtin_return_address returns the address of the call instruction
- // on the SPARC and not the return address, so we need to compensate.
- trace_buffer[0] = GetNextInstructionPc(pc);
-#else
trace_buffer[0] = pc;
-#endif
size = 1;
if (stack_top < 4096) return; // Sanity check for stack top.
// Flush register windows to memory
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp
index 3ebeac52280a3..813616467656b 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp
@@ -154,12 +154,10 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP(
®_count);
if (err != KERN_SUCCESS) {
VReport(1, "Error - unable to get registers for a thread\n");
- // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid,
- // or the thread does not exist. The other possible error case,
// MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's
// still safe to proceed.
- return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL
- : REGISTERS_UNAVAILABLE;
+ return err == MIG_ARRAY_TOO_LARGE ? REGISTERS_UNAVAILABLE
+ : REGISTERS_UNAVAILABLE_FATAL;
}
buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr));
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
index a674034b8e291..f3818526baab1 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp
@@ -86,6 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) {
}
Parse(file_contents);
+ UnmapOrDie(file_contents, contents_size);
}
bool SuppressionContext::Match(const char *str, const char *type,
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
index 6dcd8a2c58a2e..2e6f2d343fade 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h
@@ -13,8 +13,8 @@
#ifndef SANITIZER_SYMBOLIZER_INTERNAL_H
#define SANITIZER_SYMBOLIZER_INTERNAL_H
-#include "sanitizer_symbolizer.h"
#include "sanitizer_file.h"
+#include "sanitizer_symbolizer.h"
#include "sanitizer_vector.h"
namespace __sanitizer {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
index 27ed222745ec6..cc02c77bccdc9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp
@@ -11,11 +11,11 @@
// Libbacktrace implementation of symbolizer parts.
//===----------------------------------------------------------------------===//
-#include "sanitizer_platform.h"
+#include "sanitizer_symbolizer_libbacktrace.h"
#include "sanitizer_internal_defs.h"
+#include "sanitizer_platform.h"
#include "sanitizer_symbolizer.h"
-#include "sanitizer_symbolizer_libbacktrace.h"
#if SANITIZER_LIBBACKTRACE
# include "backtrace-supported.h"
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
index f4f2a036a1e71..a9c958b2d1001 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp
@@ -14,16 +14,16 @@
#include "sanitizer_platform.h"
#if SANITIZER_APPLE
-#include "sanitizer_allocator_internal.h"
-#include "sanitizer_mac.h"
-#include "sanitizer_symbolizer_mac.h"
+# include
+# include
+# include
+# include
+# include
+# include
-#include
-#include
-#include
-#include
-#include
-#include
+# include "sanitizer_allocator_internal.h"
+# include "sanitizer_mac.h"
+# include "sanitizer_symbolizer_mac.h"
namespace __sanitizer {
@@ -163,7 +163,7 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) {
uptr start_address = AddressInfo::kUnknown;
if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module,
&stack->info.file, &line, &start_address)) {
- process_ = nullptr;
+ Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr);
return false;
}
stack->info.line = (int)line;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
index 1ec0c5cad7a20..c8c10de10d03a 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp
@@ -91,7 +91,7 @@ bool RenderNeedsSymbolization(const char *format) { return false; }
// We don't support the stack_trace_format flag at all.
void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no,
uptr address, const AddressInfo *info, bool vs_style,
- const char *strip_path_prefix, const char *strip_func_prefix) {
+ const char *strip_path_prefix) {
CHECK(!RenderNeedsSymbolization(format));
buffer->append(kFormatFrame, frame_no, address);
}
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
index 7a74f925a17dd..fe532cf09c0fb 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp
@@ -13,25 +13,25 @@
#include "sanitizer_platform.h"
#if SANITIZER_POSIX && !SANITIZER_EMSCRIPTEN
-#include "sanitizer_allocator_internal.h"
-#include "sanitizer_common.h"
-#include "sanitizer_file.h"
-#include "sanitizer_flags.h"
-#include "sanitizer_internal_defs.h"
-#include "sanitizer_linux.h"
-#include "sanitizer_placement_new.h"
-#include "sanitizer_posix.h"
-#include "sanitizer_procmaps.h"
-#include "sanitizer_symbolizer_internal.h"
-#include "sanitizer_symbolizer_libbacktrace.h"
-#include "sanitizer_symbolizer_mac.h"
-
-#include // for dlsym()
-#include
-#include
-#include
-#include
-#include
+# include // for dlsym()
+# include
+# include
+# include
+# include
+# include
+
+# include "sanitizer_allocator_internal.h"
+# include "sanitizer_common.h"
+# include "sanitizer_file.h"
+# include "sanitizer_flags.h"
+# include "sanitizer_internal_defs.h"
+# include "sanitizer_linux.h"
+# include "sanitizer_placement_new.h"
+# include "sanitizer_posix.h"
+# include "sanitizer_procmaps.h"
+# include "sanitizer_symbolizer_internal.h"
+# include "sanitizer_symbolizer_libbacktrace.h"
+# include "sanitizer_symbolizer_mac.h"
// C++ demangling function, as required by Itanium C++ ABI. This is weak,
// because we do not require a C++ ABI library to be linked to a program
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
index f23fd0ada8b25..4e43baac3c5e9 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp
@@ -117,8 +117,7 @@ void ReportMmapWriteExec(int prot, int flags) {
stack->Reset();
uptr top = 0;
uptr bottom = 0;
- GET_CALLER_PC_BP_SP;
- (void)sp;
+ GET_CALLER_PC_BP;
bool fast = common_flags()->fast_unwind_on_fatal;
if (StackTrace::WillUseFastUnwind(fast)) {
GetThreadStackTopAndBottom(false, &top, &bottom);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
index ac2afe42e2692..ae2d3be19ef38 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp
@@ -14,8 +14,8 @@
#include "sanitizer_platform.h"
#if SANITIZER_WINDOWS
-#include "sanitizer_dbghelp.h"
-#include "sanitizer_symbolizer_internal.h"
+# include "sanitizer_dbghelp.h"
+# include "sanitizer_symbolizer_internal.h"
namespace __sanitizer {
@@ -292,15 +292,15 @@ static void ChooseSymbolizerTools(IntrusiveList *list,
const char *path =
user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe");
if (path) {
- VReport(2, "Using llvm-symbolizer at %spath: %s\n",
- user_path ? "user-specified " : "", path);
- list->push_back(new(*allocator) LLVMSymbolizer(path, allocator));
- } else {
if (user_path && user_path[0] == '\0') {
VReport(2, "External symbolizer is explicitly disabled.\n");
} else {
- VReport(2, "External symbolizer is not present.\n");
+ VReport(2, "Using llvm-symbolizer at %spath: %s\n",
+ user_path ? "user-specified " : "", path);
+ list->push_back(new (*allocator) LLVMSymbolizer(path, allocator));
}
+ } else {
+ VReport(2, "External symbolizer is not present.\n");
}
// Add the dbghelp based symbolizer.
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp
new file mode 100644
index 0000000000000..bddb285214085
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp
@@ -0,0 +1,94 @@
+//===-- sanitizer_thread_arg_retval.cpp -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// Tracks thread arguments and return value for leak checking.
+//===----------------------------------------------------------------------===//
+
+#include "sanitizer_thread_arg_retval.h"
+
+#include "sanitizer_placement_new.h"
+
+namespace __sanitizer {
+
+void ThreadArgRetval::CreateLocked(uptr thread, bool detached,
+ const Args& args) {
+ CheckLocked();
+ Data& t = data_[thread];
+ t = {};
+ t.gen = gen_++;
+ t.detached = detached;
+ t.args = args;
+}
+
+ThreadArgRetval::Args ThreadArgRetval::GetArgs(uptr thread) const {
+ __sanitizer::Lock lock(&mtx_);
+ auto t = data_.find(thread);
+ CHECK(t);
+ if (t->second.done)
+ return {};
+ return t->second.args;
+}
+
+void ThreadArgRetval::Finish(uptr thread, void* retval) {
+ __sanitizer::Lock lock(&mtx_);
+ auto t = data_.find(thread);
+ if (!t)
+ return;
+ if (t->second.detached) {
+ // Retval of detached thread connot be retrieved.
+ data_.erase(t);
+ return;
+ }
+ t->second.done = true;
+ t->second.args.arg_retval = retval;
+}
+
+u32 ThreadArgRetval::BeforeJoin(uptr thread) const {
+ __sanitizer::Lock lock(&mtx_);
+ auto t = data_.find(thread);
+ CHECK(t);
+ CHECK(!t->second.detached);
+ return t->second.gen;
+}
+
+void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) {
+ __sanitizer::Lock lock(&mtx_);
+ auto t = data_.find(thread);
+ if (!t || gen != t->second.gen) {
+ // Thread was reused and erased by any other event.
+ return;
+ }
+ CHECK(!t->second.detached);
+ data_.erase(t);
+}
+
+void ThreadArgRetval::DetachLocked(uptr thread) {
+ CheckLocked();
+ auto t = data_.find(thread);
+ CHECK(t);
+ CHECK(!t->second.detached);
+ if (t->second.done) {
+ // We can't retrive retval after detached thread finished.
+ data_.erase(t);
+ return;
+ }
+ t->second.detached = true;
+}
+
+void ThreadArgRetval::GetAllPtrsLocked(InternalMmapVector* ptrs) {
+ CheckLocked();
+ CHECK(ptrs);
+ data_.forEach([&](DenseMap::value_type& kv) -> bool {
+ ptrs->push_back((uptr)kv.second.args.arg_retval);
+ return true;
+ });
+}
+
+} // namespace __sanitizer
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h
new file mode 100644
index 0000000000000..c77021beb67d1
--- /dev/null
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h
@@ -0,0 +1,116 @@
+//===-- sanitizer_thread_arg_retval.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is shared between sanitizer tools.
+//
+// Tracks thread arguments and return value for leak checking.
+//===----------------------------------------------------------------------===//
+
+#ifndef SANITIZER_THREAD_ARG_RETVAL_H
+#define SANITIZER_THREAD_ARG_RETVAL_H
+
+#include "sanitizer_common.h"
+#include "sanitizer_dense_map.h"
+#include "sanitizer_list.h"
+#include "sanitizer_mutex.h"
+
+namespace __sanitizer {
+
+// Primary goal of the class is to keep alive arg and retval pointer for leak
+// checking. However it can be used to pass those pointer into wrappers used by
+// interceptors. The difference from ThreadRegistry/ThreadList is that this
+// class keeps data up to the detach or join, as exited thread still can be
+// joined to retrive retval. ThreadRegistry/ThreadList can discard exited
+// threads immediately.
+class SANITIZER_MUTEX ThreadArgRetval {
+ public:
+ struct Args {
+ void* (*routine)(void*);
+ void* arg_retval; // Either arg or retval.
+ };
+ void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); }
+ void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); }
+ void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); }
+
+ // Wraps pthread_create or similar. We need to keep object locked, to
+ // prevent child thread from proceeding without thread handle.
+ template
+ void Create(bool detached, const Args& args, const CreateFn& fn) {
+ // No need to track detached threads with no args, but we will to do as it's
+ // not expensive and less edge-cases.
+ __sanitizer::Lock lock(&mtx_);
+ if (uptr thread = fn())
+ CreateLocked(thread, detached, args);
+ }
+
+ // Returns thread arg and routine.
+ Args GetArgs(uptr thread) const;
+
+ // Mark thread as done and stores retval or remove if detached. Should be
+ // called by the thread.
+ void Finish(uptr thread, void* retval);
+
+ // Mark thread as detached or remove if done.
+ template
+ void Detach(uptr thread, const DetachFn& fn) {
+ // Lock to prevent re-use of the thread between fn() and DetachLocked()
+ // calls.
+ __sanitizer::Lock lock(&mtx_);
+ if (fn())
+ DetachLocked(thread);
+ }
+
+ // Joins the thread.
+ template
+ void Join(uptr thread, const JoinFn& fn) {
+ // Remember internal id of the thread to prevent re-use of the thread
+ // between fn() and AfterJoin() calls. Locking JoinFn, like in
+ // Detach(), implementation can cause deadlock.
+ auto gen = BeforeJoin(thread);
+ if (fn())
+ AfterJoin(thread, gen);
+ }
+
+ // Returns all arg and retval which are considered alive.
+ void GetAllPtrsLocked(InternalMmapVector* ptrs);
+
+ uptr size() const {
+ __sanitizer::Lock lock(&mtx_);
+ return data_.size();
+ }
+
+ // FIXME: Add fork support. Expected users of the class are sloppy with forks
+ // anyway. We likely should lock/unlock the object to avoid deadlocks, and
+ // erase all but the current threads, so we can detect leaked arg or retval in
+ // child process.
+
+ // FIXME: Add cancelation support. Now if a thread was canceled, the class
+ // will keep pointers alive forever, missing leaks caused by cancelation.
+
+ private:
+ struct Data {
+ Args args;
+ u32 gen; // Avoid collision if thread id re-used.
+ bool detached;
+ bool done;
+ };
+
+ void CreateLocked(uptr thread, bool detached, const Args& args);
+ u32 BeforeJoin(uptr thread) const;
+ void AfterJoin(uptr thread, u32 gen);
+ void DetachLocked(uptr thread);
+
+ mutable Mutex mtx_;
+
+ DenseMap data_;
+ u32 gen_ = 0;
+};
+
+} // namespace __sanitizer
+
+#endif // SANITIZER_THREAD_ARG_RETVAL_H
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
index 278f6defca95f..741e0731c4155 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp
@@ -335,7 +335,7 @@ void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) {
ThreadContextBase *ThreadRegistry::QuarantinePop() {
if (invalid_threads_.size() == 0)
- return 0;
+ return nullptr;
ThreadContextBase *tctx = invalid_threads_.front();
invalid_threads_.pop_front();
return tctx;
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
index b13e2dc9e3327..252979f1c2baa 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp
@@ -12,6 +12,7 @@
#include "sanitizer_tls_get_addr.h"
+#include "sanitizer_allocator_interface.h"
#include "sanitizer_atomic.h"
#include "sanitizer_flags.h"
#include "sanitizer_platform_interceptors.h"
@@ -26,13 +27,6 @@ struct TlsGetAddrParam {
uptr offset;
};
-// Glibc starting from 2.19 allocates tls using __signal_safe_memalign,
-// which has such header.
-struct Glibc_2_19_tls_header {
- uptr size;
- uptr start;
-};
-
// This must be static TLS
__attribute__((tls_model("initial-exec")))
static __thread DTLS dtls;
@@ -108,6 +102,14 @@ static const uptr kDtvOffset = 0x800;
static const uptr kDtvOffset = 0;
#endif
+extern "C" {
+SANITIZER_WEAK_ATTRIBUTE
+uptr __sanitizer_get_allocated_size(const void *p);
+
+SANITIZER_WEAK_ATTRIBUTE
+const void *__sanitizer_get_allocated_begin(const void *p);
+}
+
DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
uptr static_tls_begin, uptr static_tls_end) {
if (!common_flags()->intercept_tls_get_addr) return 0;
@@ -125,19 +127,18 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res,
atomic_load(&number_of_live_dtls, memory_order_relaxed));
if (dtls.last_memalign_ptr == tls_beg) {
tls_size = dtls.last_memalign_size;
- VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={0x%zx,0x%zx}\n",
+ VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n",
tls_beg, tls_size);
} else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) {
// This is the static TLS block which was initialized / unpoisoned at thread
// creation.
VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg);
tls_size = 0;
- } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) {
- // We may want to check gnu_get_libc_version().
- Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1;
- tls_size = header->size;
- tls_beg = header->start;
- VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={0x%zx 0x%zx}\n",
+ } else if (const void *start =
+ __sanitizer_get_allocated_begin((void *)tls_beg)) {
+ tls_beg = (uptr)start;
+ tls_size = __sanitizer_get_allocated_size(start);
+ VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n",
tls_beg, tls_size);
} else {
VReport(2, "__tls_get_addr: Can't guess glibc version\n");
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
index a599c0bbc75cc..0ddab61deb102 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h
@@ -12,16 +12,24 @@
// the lack of interface that would tell us about the Dynamic TLS (DTLS).
// https://sourceware.org/bugzilla/show_bug.cgi?id=16291
//
-// The matters get worse because the glibc implementation changed between
-// 2.18 and 2.19:
-// https://groups.google.com/forum/#!topic/address-sanitizer/BfwYD8HMxTM
-//
-// Before 2.19, every DTLS chunk is allocated with __libc_memalign,
+// Before 2.25: every DTLS chunk is allocated with __libc_memalign,
// which we intercept and thus know where is the DTLS.
-// Since 2.19, DTLS chunks are allocated with __signal_safe_memalign,
-// which is an internal function that wraps a mmap call, neither of which
-// we can intercept. Luckily, __signal_safe_memalign has a simple parseable
-// header which we can use.
+//
+// Since 2.25: DTLS chunks are allocated with malloc. We could co-opt
+// the malloc interceptor to keep track of the last allocation, similar
+// to how we handle __libc_memalign; however, this adds some overhead
+// (since malloc, unlike __libc_memalign, is commonly called), and
+// requires care to avoid false negatives for LeakSanitizer.
+// Instead, we rely on our internal allocators - which keep track of all
+// its allocations - to determine if an address points to a malloc
+// allocation.
+//
+// There exists a since-deprecated version of Google's internal glibc fork
+// that used __signal_safe_memalign. DTLS_on_tls_get_addr relied on a
+// heuristic check (is the allocation 16 bytes from the start of a page
+// boundary?), which was sometimes erroneous:
+// https://bugs.chromium.org/p/chromium/issues/detail?id=1275223#c15
+// Since that check has no practical use anymore, we have removed it.
//
//===----------------------------------------------------------------------===//
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
index 72f025a7d307a..6a8e82e2e213c 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
@@ -139,13 +139,7 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
if (to_pop == 0 && size > 1)
to_pop = 1;
PopStackFrames(to_pop);
-#if defined(__GNUC__) && defined(__sparc__)
- // __builtin_return_address returns the address of the call instruction
- // on the SPARC and not the return address, so we need to compensate.
- trace_buffer[0] = GetNextInstructionPc(pc);
-#else
trace_buffer[0] = pc;
-#endif
}
void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) {
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
index e0568c9b62d5e..06e496523eeaa 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp
@@ -362,6 +362,11 @@ bool MprotectReadOnly(uptr addr, uptr size) {
return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection);
}
+bool MprotectReadWrite(uptr addr, uptr size) {
+ DWORD old_protection;
+ return VirtualProtect((LPVOID)addr, size, PAGE_READWRITE, &old_protection);
+}
+
void ReleaseMemoryPagesToOS(uptr beg, uptr end) {
uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()),
end_aligned = RoundDownTo(end, GetPageSizeCached());
@@ -718,13 +723,24 @@ void ListOfModules::fallbackInit() { clear(); }
// atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers).
InternalMmapVectorNoCtor atexit_functions;
-int Atexit(void (*function)(void)) {
+static int queueAtexit(void (*function)(void)) {
atexit_functions.push_back(function);
return 0;
}
+// If Atexit() is being called after RunAtexit() has already been run, it needs
+// to be able to call atexit() directly. Here we use a function ponter to
+// switch out its behaviour.
+// An example of where this is needed is the asan_dynamic runtime on MinGW-w64.
+// On this environment, __asan_init is called during global constructor phase,
+// way after calling the .CRT$XID initializer.
+static int (*volatile queueOrCallAtExit)(void (*)(void)) = &queueAtexit;
+
+int Atexit(void (*function)(void)) { return queueOrCallAtExit(function); }
+
static int RunAtexit() {
TraceLoggingUnregister(g_asan_provider);
+ queueOrCallAtExit = &atexit;
int ret = 0;
for (uptr i = 0; i < atexit_functions.size(); ++i) {
ret |= atexit(atexit_functions[i]);
diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h
index 48c73c4c98ad5..639d91a2edaec 100644
--- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h
+++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h
@@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init();
// which isn't a big deal.
#define INTERCEPT_LIBRARY_FUNCTION(name) \
extern "C" void name(); \
- INTERCEPT_OR_DIE(WRAPPER_NAME(name), name)
+ INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name)
// Use these macros for functions that could be called before __dll_thunk_init()
// is executed and don't lead to errors if defined (free, malloc, etc).
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp
index 8c6651b272e26..673a4d3dbd67c 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp
@@ -214,7 +214,12 @@ static void RenderText(InternalScopedString *Buffer, const char *Message,
// printf, and stop using snprintf here.
char FloatBuffer[32];
#if SANITIZER_WINDOWS
- sprintf_s(FloatBuffer, sizeof(FloatBuffer), "%Lg", (long double)A.Float);
+ // On MSVC platforms, long doubles are equal to regular doubles.
+ // In MinGW environments on x86, long doubles are 80 bit, but here,
+ // we're calling an MS CRT provided printf function which considers
+ // long doubles to be 64 bit. Just cast the float value to a regular
+ // double to avoid the potential ambiguity in MinGW mode.
+ sprintf_s(FloatBuffer, sizeof(FloatBuffer), "%g", (double)A.Float);
#else
snprintf(FloatBuffer, sizeof(FloatBuffer), "%Lg", (long double)A.Float);
#endif
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp
index a0b869e7ede1b..85342711e318e 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp
@@ -896,4 +896,39 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data,
Die();
}
+static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data,
+ ValueHandle Function,
+ ReportOptions Opts) {
+ SourceLocation CallLoc = Data->Loc.acquire();
+ ErrorType ET = ErrorType::FunctionTypeMismatch;
+ if (ignoreReport(CallLoc, Opts, ET))
+ return true;
+
+ ScopedReport R(Opts, CallLoc, ET);
+
+ SymbolizedStackHolder FLoc(getSymbolizedLocation(Function));
+ const char *FName = FLoc.get()->info.function;
+ if (!FName)
+ FName = "(unknown)";
+
+ Diag(CallLoc, DL_Error, ET,
+ "call to function %0 through pointer to incorrect function type %1")
+ << FName << Data->Type;
+ Diag(FLoc, DL_Note, ET, "%0 defined here") << FName;
+ return true;
+}
+
+void __ubsan::__ubsan_handle_function_type_mismatch(
+ FunctionTypeMismatchData *Data, ValueHandle Function) {
+ GET_REPORT_OPTIONS(false);
+ handleFunctionTypeMismatch(Data, Function, Opts);
+}
+
+void __ubsan::__ubsan_handle_function_type_mismatch_abort(
+ FunctionTypeMismatchData *Data, ValueHandle Function) {
+ GET_REPORT_OPTIONS(true);
+ if (handleFunctionTypeMismatch(Data, Function, Opts))
+ Die();
+}
+
#endif // CAN_SANITIZE_UB
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h
index 219fb15de55fe..3bd5046de3d7d 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h
@@ -231,6 +231,17 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type(
CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable,
ReportOptions Opts);
+struct FunctionTypeMismatchData {
+ SourceLocation Loc;
+ const TypeDescriptor &Type;
+};
+
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data,
+ ValueHandle Val);
+extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
+__ubsan_handle_function_type_mismatch_abort(FunctionTypeMismatchData *Data,
+ ValueHandle Val);
}
#endif // UBSAN_HANDLERS_H
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp
index 0317a3d1428c8..206a0bb485a9c 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp
@@ -156,50 +156,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable,
Diag(Loc, DL_Note, ET, "check failed in %0, vtable located in %1")
<< SrcModule << DstModule;
}
-
-static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data,
- ValueHandle Function,
- ValueHandle calleeRTTI,
- ValueHandle fnRTTI, ReportOptions Opts) {
- if (checkTypeInfoEquality(reinterpret_cast(calleeRTTI),
- reinterpret_cast(fnRTTI)))
- return false;
-
- SourceLocation CallLoc = Data->Loc.acquire();
- ErrorType ET = ErrorType::FunctionTypeMismatch;
-
- if (ignoreReport(CallLoc, Opts, ET))
- return true;
-
- ScopedReport R(Opts, CallLoc, ET);
-
- SymbolizedStackHolder FLoc(getSymbolizedLocation(Function));
- const char *FName = FLoc.get()->info.function;
- if (!FName)
- FName = "(unknown)";
-
- Diag(CallLoc, DL_Error, ET,
- "call to function %0 through pointer to incorrect function type %1")
- << FName << Data->Type;
- Diag(FLoc, DL_Note, ET, "%0 defined here") << FName;
- return true;
-}
-
-void __ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data,
- ValueHandle Function,
- ValueHandle calleeRTTI,
- ValueHandle fnRTTI) {
- GET_REPORT_OPTIONS(false);
- handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts);
-}
-
-void __ubsan_handle_function_type_mismatch_v1_abort(
- FunctionTypeMismatchData *Data, ValueHandle Function,
- ValueHandle calleeRTTI, ValueHandle fnRTTI) {
- GET_REPORT_OPTIONS(true);
- if (handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts))
- Die();
-}
} // namespace __ubsan
#endif // CAN_SANITIZE_UB
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h
index fd534c2573f6d..71695cbdc090f 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h
@@ -33,22 +33,6 @@ void __ubsan_handle_dynamic_type_cache_miss(
extern "C" SANITIZER_INTERFACE_ATTRIBUTE
void __ubsan_handle_dynamic_type_cache_miss_abort(
DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash);
-
-struct FunctionTypeMismatchData {
- SourceLocation Loc;
- const TypeDescriptor &Type;
-};
-
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
-__ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data,
- ValueHandle Val,
- ValueHandle calleeRTTI,
- ValueHandle fnRTTI);
-extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
-__ubsan_handle_function_type_mismatch_v1_abort(FunctionTypeMismatchData *Data,
- ValueHandle Val,
- ValueHandle calleeRTTI,
- ValueHandle fnRTTI);
}
#endif // UBSAN_HANDLERS_CXX_H
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc b/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc
index 94337d85017b4..cb27feb5d7e99 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc
@@ -21,8 +21,8 @@ INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss)
INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort)
INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow)
INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort)
-INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1)
-INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1_abort)
+INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch)
+INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort)
INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion)
INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion_abort)
INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin)
diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
index 8d980e1a26391..fec9ac2e032d2 100644
--- a/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
+++ b/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
@@ -34,7 +34,12 @@ void InitializeDeadlySignals() {}
#else
+namespace __ubsan {
+void InitializeDeadlySignals();
+} // namespace __ubsan
+
#define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
+#define SIGNAL_INTERCEPTOR_ENTER() __ubsan::InitializeDeadlySignals()
#include "sanitizer_common/sanitizer_signal_interceptors.inc"
// TODO(yln): Temporary workaround. Will be removed.
diff --git a/system/lib/libcxx/CREDITS.TXT b/system/lib/libcxx/CREDITS.TXT
index cd5bc08a60fc1..aa3c8cf1a8c6e 100644
--- a/system/lib/libcxx/CREDITS.TXT
+++ b/system/lib/libcxx/CREDITS.TXT
@@ -92,6 +92,11 @@ E: stl@microsoft.com
E: stl@nuwen.net
D: Implemented floating-point to_chars.
+N: Damien Lebrun-Grandie
+E: dalg24@gmail.com
+E: lebrungrandt@ornl.gov
+D: Implementation of mdspan.
+
N: Microsoft Corporation
D: Contributed floating-point to_chars.
@@ -149,6 +154,10 @@ N: Stephan Tolksdorf
E: st@quanttec.com
D: Minor fix
+N: Christian Trott
+E: crtrott@sandia.gov
+D: Implementation of mdspan.
+
N: Ruben Van Boxem
E: vanboxem dot ruben at gmail dot com
D: Initial Windows patches.
diff --git a/system/lib/libcxx/include/__algorithm/adjacent_find.h b/system/lib/libcxx/include/__algorithm/adjacent_find.h
index 30df4a976f3d8..7819e2cf49b9f 100644
--- a/system/lib/libcxx/include/__algorithm/adjacent_find.h
+++ b/system/lib/libcxx/include/__algorithm/adjacent_find.h
@@ -20,6 +20,9 @@
# pragma GCC system_header
#endif
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
_LIBCPP_BEGIN_NAMESPACE_STD
template
@@ -50,4 +53,6 @@ adjacent_find(_ForwardIterator __first, _ForwardIterator __last) {
_LIBCPP_END_NAMESPACE_STD
+_LIBCPP_POP_MACROS
+
#endif // _LIBCPP___ALGORITHM_ADJACENT_FIND_H
diff --git a/system/lib/libcxx/include/__algorithm/all_of.h b/system/lib/libcxx/include/__algorithm/all_of.h
index 284c34ffcda9b..237f8495c645f 100644
--- a/system/lib/libcxx/include/__algorithm/all_of.h
+++ b/system/lib/libcxx/include/__algorithm/all_of.h
@@ -19,7 +19,7 @@
_LIBCPP_BEGIN_NAMESPACE_STD
template
-_LIBCPP_NODISCARD_EXT inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
+_LIBCPP_NODISCARD_EXT inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
all_of(_InputIterator __first, _InputIterator __last, _Predicate __pred) {
for (; __first != __last; ++__first)
if (!__pred(*__first))
diff --git a/system/lib/libcxx/include/__algorithm/binary_search.h b/system/lib/libcxx/include/__algorithm/binary_search.h
index 8f958c2c1ad89..0c8f5545e066e 100644
--- a/system/lib/libcxx/include/__algorithm/binary_search.h
+++ b/system/lib/libcxx/include/__algorithm/binary_search.h
@@ -37,8 +37,7 @@ _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
bool
binary_search(_ForwardIterator __first, _ForwardIterator __last, const _Tp& __value)
{
- return std::binary_search(__first, __last, __value,
- __less::value_type, _Tp>());
+ return std::binary_search(__first, __last, __value, __less<>());
}
_LIBCPP_END_NAMESPACE_STD
diff --git a/system/lib/libcxx/include/__algorithm/clamp.h b/system/lib/libcxx/include/__algorithm/clamp.h
index 30ddbdce64a92..31deb4fd94aa1 100644
--- a/system/lib/libcxx/include/__algorithm/clamp.h
+++ b/system/lib/libcxx/include/__algorithm/clamp.h
@@ -19,14 +19,14 @@
_LIBCPP_BEGIN_NAMESPACE_STD
-#if _LIBCPP_STD_VER > 14
+#if _LIBCPP_STD_VER >= 17
template
_LIBCPP_NODISCARD_EXT inline
_LIBCPP_INLINE_VISIBILITY constexpr
const _Tp&
clamp(const _Tp& __v, const _Tp& __lo, const _Tp& __hi, _Compare __comp)
{
- _LIBCPP_ASSERT(!__comp(__hi, __lo), "Bad bounds passed to std::clamp");
+ _LIBCPP_ASSERT_UNCATEGORIZED(!__comp(__hi, __lo), "Bad bounds passed to std::clamp");
return __comp(__v, __lo) ? __lo : __comp(__hi, __v) ? __hi : __v;
}
@@ -37,7 +37,7 @@ _LIBCPP_INLINE_VISIBILITY constexpr
const _Tp&
clamp(const _Tp& __v, const _Tp& __lo, const _Tp& __hi)
{
- return _VSTD::clamp(__v, __lo, __hi, __less<_Tp>());
+ return _VSTD::clamp(__v, __lo, __hi, __less<>());
}
#endif
diff --git a/system/lib/libcxx/include/__algorithm/comp.h b/system/lib/libcxx/include/__algorithm/comp.h
index af8eb7b5d76b1..9474536615ffb 100644
--- a/system/lib/libcxx/include/__algorithm/comp.h
+++ b/system/lib/libcxx/include/__algorithm/comp.h
@@ -10,6 +10,8 @@
#define _LIBCPP___ALGORITHM_COMP_H
#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/predicate_traits.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -24,41 +26,20 @@ struct __equal_to {
}
};
-template
-struct __less
-{
- _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
- bool operator()(const _T1& __x, const _T1& __y) const {return __x < __y;}
+template
+struct __is_trivial_equality_predicate<__equal_to, _Lhs, _Rhs> : true_type {};
- _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
- bool operator()(const _T1& __x, const _T2& __y) const {return __x < __y;}
+// The definition is required because __less is part of the ABI, but it's empty
+// because all comparisons should be transparent.
+template
+struct __less {};
- _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
- bool operator()(const _T2& __x, const _T1& __y) const {return __x < __y;}
-
- _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX14
- bool operator()(const _T2& __x, const _T2& __y) const {return __x < __y;}
-};
-
-template