From 4b3a197dc2eb0e1c83b847b250f448aa765ad4d5 Mon Sep 17 00:00:00 2001 From: Mike Ash Date: Fri, 19 Jul 2024 16:34:19 -0400 Subject: [PATCH] [Runtime] Support type descriptor map in LibPrespecialized. The descriptor map is keyed by a simplified mangling that canonicalizes the differences that we accept in _contextDescriptorMatchesMangling, such as the ability to specify any kind of type with an OtherNominalType node. This simplified mangling is not necessarily unique, but we use _contextDescriptorMatchesMangling for the final equality checking when looking up entries in the map, so occasional collisions are acceptable and get resolved when probing the table. The table is meant to be comprehensive, so it includes all descriptors that can be looked up by name, and a negative result means the descriptor does not exist in the shared cache. We add a flag to the options that can mark it as non-definitive in case we ever need to degrade this, and fall back to a full search after a negative result. The map encompasses the entire shared cache but we need to reject lookups for types in images that aren't loaded. The map includes an image index which allows us to cheaply query whether a given descriptor is in a loaded image or not, so we can ignore ones which are not. TypeMetadataPrivateState now has a separate sections array for sections within the shared cache. _searchTypeMetadataRecords consults the map first. If no result is found in the map and the map is marked as comprehensive, then only the sections outside the shared cache need to be scanned. Replace the SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED environment variable with one specifically for metadata and one for descriptor lookup so they can be controlled independently. Also add SWIFT_DEBUG_VALIDATE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP which consults the map and does the full scan, and ensures they produce the same result, for debugging purposes. Enhance the environment variable code to track whether a variable was set at all. This allows SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED to override the default in either direction. Remove the disablePrespecializedMetadata global and instead modify the mapConfiguration to disable prespecialized metadata when an image is loaded that overrides one in the shared cache. rdar://113059233 --- include/swift/Runtime/EnvironmentVariables.h | 14 +- include/swift/Runtime/LibPrespecialized.h | 157 ++++++++- include/swift/Runtime/PrebuiltStringMap.h | 209 ++++++++--- .../public/runtime/EnvironmentVariables.cpp | 32 +- .../public/runtime/EnvironmentVariables.def | 11 +- stdlib/public/runtime/LibPrespecialized.cpp | 325 +++++++++++++----- stdlib/public/runtime/MetadataLookup.cpp | 186 ++++++++-- unittests/runtime/PrebuiltStringMap.cpp | 73 +++- 8 files changed, 822 insertions(+), 185 deletions(-) diff --git a/include/swift/Runtime/EnvironmentVariables.h b/include/swift/Runtime/EnvironmentVariables.h index de4f799beb586..c0c4c1cdcfcfc 100644 --- a/include/swift/Runtime/EnvironmentVariables.h +++ b/include/swift/Runtime/EnvironmentVariables.h @@ -30,14 +30,24 @@ extern swift::once_t initializeToken; using string = const char *; // Declare backing variables. -#define VARIABLE(name, type, defaultValue, help) extern type name ## _variable; +#define VARIABLE(name, type, defaultValue, help) \ + extern type name##_variable; \ + extern bool name##_isSet_variable; #include "../../../stdlib/public/runtime/EnvironmentVariables.def" -// Define getter functions. +// Define getter functions. This creates one function with the same name as the +// variable which returns the value set for that variable, and second function +// ending in _isSet which returns a boolean indicating whether the variable was +// set at all, to allow detecting when the variable was explicitly set to the +// same value as the default. #define VARIABLE(name, type, defaultValue, help) \ inline type name() { \ swift::once(initializeToken, initialize, nullptr); \ return name##_variable; \ + } \ + inline bool name##_isSet() { \ + swift::once(initializeToken, initialize, nullptr); \ + return name##_isSet_variable; \ } #include "../../../stdlib/public/runtime/EnvironmentVariables.def" diff --git a/include/swift/Runtime/LibPrespecialized.h b/include/swift/Runtime/LibPrespecialized.h index b9523abed7237..1a74bbf34cfd2 100644 --- a/include/swift/Runtime/LibPrespecialized.h +++ b/include/swift/Runtime/LibPrespecialized.h @@ -20,6 +20,7 @@ #include "PrebuiltStringMap.h" #include "swift/ABI/Metadata.h" #include "swift/ABI/TargetLayout.h" +#include "swift/Demangling/Demangler.h" #define LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME "_swift_prespecializationsData" @@ -36,23 +37,44 @@ struct LibPrespecializedData { typename Runtime::StoredSize optionFlags; + TargetPointer descriptorMap; + // Existing fields are above, add new fields below this point. + // The major/minor version numbers for this version of the struct. static constexpr uint32_t currentMajorVersion = 1; - static constexpr uint32_t currentMinorVersion = 3; + static constexpr uint32_t currentMinorVersion = 4; + // Version numbers where various fields were introduced. static constexpr uint32_t minorVersionWithDisabledProcessesTable = 2; static constexpr uint32_t minorVersionWithPointerKeyedMetadataMap = 3; static constexpr uint32_t minorVersionWithOptionFlags = 3; + static constexpr uint32_t minorVersionWithDescriptorMap = 4; // Option flags values. enum : typename Runtime::StoredSize { // When this flag is set, the runtime should default to using the // pointer-keyed table. When not set, default to using the name-keyed table. OptionFlagDefaultToPointerKeyedMap = 1ULL << 0, + + // When this flag is set, the runtime should default to using the descriptor + // map. When not set, default to turning off the descriptor map. + OptionFlagDescriptorMapDefaultOn = 1ULL << 1, + + // When this flag is set, descriptorMap is not comprehensive, meaning that + // a negative lookup result is not a definitive failure. + OptionFlagDescriptorMapNotComprehensive = 1ULL << 2, }; - // Helpers for retrieving the metadata map in-process. + // Helpers for safely retrieving various fields. Helpers return 0 or NULL if + // the version number indicates that the field is not present. + + typename Runtime::StoredSize getOptionFlags() const { + if (minorVersion < minorVersionWithOptionFlags) + return 0; + return optionFlags; + } + static bool stringIsNull(const char *str) { return str == nullptr; } using MetadataMap = PrebuiltStringMap; @@ -73,18 +95,141 @@ struct LibPrespecializedData { return pointerKeyedMetadataMap; } - typename Runtime::StoredSize getOptionFlags() const { - if (minorVersion < minorVersionWithOptionFlags) - return 0; - return optionFlags; + using DescriptorMap = + PrebuiltAuxDataImplicitStringMap, + uint16_t>; + + const DescriptorMap *getDescriptorMap() const { + if (minorVersion < minorVersionWithDescriptorMap) + return nullptr; + return reinterpret_cast(descriptorMap); } }; +enum class LibPrespecializedLookupResult { + // We found something. + Found, + + // We didn't find anything, and we know it's not in the shared cache. + DefinitiveNotFound, + + // We didn't find anything, but we couldn't rule out the shared cache. Caller + // must do a full search. + NonDefinitiveNotFound, +}; + const LibPrespecializedData *getLibPrespecializedData(); + Metadata *getLibPrespecializedMetadata(const TypeContextDescriptor *description, const void *const *arguments); void libPrespecializedImageLoaded(); +std::pair +getLibPrespecializedTypeDescriptor(Demangle::NodePointer node); + +/// Given the demangling referring to a particular descriptor, build the +/// canonical simplified version of the demangling that's used for the keys in +/// the descriptorMap. We copy across Extension and Module nodes. Type nodes are +/// all normalized to be OtherNominalType to allow for the runtime allowing +/// type kind mismatches on imported C types in certain cases. Other nodes are +/// skipped. +/// +/// The runtime always searches through duplicates in the table, and uses its +/// own matching on all candidates, so the simplified demangling is allowed to +/// be simplified to the point of having different descriptors sometimes produce +/// the same demangling. +static inline Demangle::NodePointer +buildSimplifiedDescriptorDemangling(Demangle::NodePointer node, + Demangle::Demangler &dem) { + // The node that will be returned to the caller. + Demangle::NodePointer result = nullptr; + + // The bottommost node in the result that we've generated. Additional nodes + // are added as children to this one. + Demangle::NodePointer resultBottom = nullptr; + + // The current node that we're iterating over in the input node tree. + Demangle::NodePointer current = node; + + using Kind = Demangle::Node::Kind; + + // Helper to add a new node to the result. This sets `result` to the node if + // it hasn't already been set (indicating this is the topmost node), and adds + // the node as a child to `resultBottom` otherwise. `resultBottom` is updated + // to point to the new node. + auto addNode = [&](Demangle::NodePointer newNode) { + if (!result) { + result = newNode; + } else { + if (resultBottom->getKind() == Kind::Extension) { + resultBottom->addChild(newNode, dem); + } else { + // Shift the Identifier down, insert before it. + resultBottom->addChild(resultBottom->getFirstChild(), dem); + resultBottom->replaceChild(0, newNode); + } + } + resultBottom = newNode; + }; + + // Walk down the input node tree. + while (current) { + switch (current->getKind()) { + case Kind::Extension: { + // Extensions are copied across. The new extension node has the module + // from the original, and the second child will be added as we traverse + // the next node in the tree. + auto copy = dem.createNode(Kind::Extension); + auto module = current->getChild(0); + if (module == nullptr || module->getKind() != Kind::Module) + return nullptr; + copy->addChild(module, dem); + addNode(copy); + current = current->getChild(1); + break; + } + case Kind::Module: { + // Module contents are always in the form we want, so we can incorporate + // this node verbatim and terminate the walk. + addNode(current); + current = nullptr; + break; + } + case Kind::Protocol: { + // Bring Protocol nodes across verbatim, there's no fuzzy matching. + addNode(current); + current = nullptr; + break; + } + case Kind::OpaqueType: + case Kind::Class: + case Kind::Structure: + case Kind::Enum: + case Kind::TypeAlias: + case Kind::OtherNominalType: { + // Type nodes are copied across with the kind always set to + // OtherNominalType. + auto copy = dem.createNode(Kind::OtherNominalType); + auto identifier = current->getChild(1); + if (identifier == nullptr || identifier->getKind() != Kind::Identifier) + return nullptr; + copy->addChild(identifier, dem); + addNode(copy); + current = current->getChild(0); + break; + } + + default: + // If we don't know about this node, continue the walk with its first + // child. + current = current->getFirstChild(); + break; + } + } + + return result; +} + } // namespace swift // Validate the prespecialized metadata map by building each entry dynamically diff --git a/include/swift/Runtime/PrebuiltStringMap.h b/include/swift/Runtime/PrebuiltStringMap.h index 85dc55a0b4ffa..7051924ed6fbe 100644 --- a/include/swift/Runtime/PrebuiltStringMap.h +++ b/include/swift/Runtime/PrebuiltStringMap.h @@ -17,44 +17,18 @@ #include #include #include +#include namespace swift { -/// A map that can be pre-built out of process. Uses a fixed hash function with -/// no per-process seeding to ensure consistent hashes between builder and user. -/// -/// The elements are tail allocated. `byteSize` can be used to calculate the -/// amount of memory needed. The memory must be initialized with all string -/// values set to null. StringTy is opaque for insertion, except for using the -/// provided stringIsNull function to check for null values. -template -struct PrebuiltStringMap { +struct PrebuiltStringMapBase { uint64_t arraySize; - struct ArrayElement { - StringTy key; - ElemTy value; - }; - - ArrayElement *array() { - uintptr_t start = (uintptr_t)(&arraySize + 1); - return (ArrayElement *)start; - } - - const ArrayElement *array() const { - uintptr_t start = (uintptr_t)(&arraySize + 1); - return (ArrayElement *)start; - } - - static size_t byteSize(uint64_t arraySize) { - return sizeof(PrebuiltStringMap) + sizeof(ArrayElement) * arraySize; - } - /// Construct an empty map. Must be constructed in memory at least as large as /// byteSize(arraySize). The map can hold at most arraySize-1 values. /// Attempting to insert more than that will result in fatal errors when /// inserting or retrieving values. - PrebuiltStringMap(uint64_t arraySize) : arraySize(arraySize) {} + PrebuiltStringMapBase(uint64_t arraySize) : arraySize(arraySize) {} // Based on MurmurHash2 uint64_t hash(const void *data, size_t len) const { @@ -116,17 +90,17 @@ struct PrebuiltStringMap { return hash; } - /// Perform the search portion of an insertion operation. Returns a pointer to - /// the element where string is to be inserted. The caller is responsible for - /// initializing the element to contain the string/value. It is assumed that - /// the key does not already exist in the map. If it does exist, this will - /// insert a useless duplicate. - ArrayElement *insert(const void *string, size_t len) { + /// Search for a matching entry in the map. `isMatch` is called with a + /// candidate index and returns true if there is a match at that index. + template + std::optional findIndex(const void *string, size_t len, + const IsMatch &isMatch) const { uint64_t hashValue = hash(string, len); + size_t index = hashValue % arraySize; size_t numSearched = 0; - while (!stringIsNull(array()[index].key)) { + while (!isMatch(index)) { index = index + 1; if (index >= arraySize) index = 0; @@ -134,12 +108,58 @@ struct PrebuiltStringMap { numSearched++; if (numSearched > arraySize) { assert(false && - "Could not find empty element in PrebuiltStringMap::insert"); - return nullptr; + "Could not find match in PrebuiltStringMapBase::findIndex"); + return std::nullopt; } } - return &array()[index]; + return index; + } +}; + +/// A map that can be pre-built out of process. Uses a fixed hash function with +/// no per-process seeding to ensure consistent hashes between builder and user. +/// +/// The elements are tail allocated. `byteSize` can be used to calculate the +/// amount of memory needed. The memory must be initialized with all string +/// values set to null. StringTy is opaque for insertion, except for using the +/// provided stringIsNull function to check for null values. +template +struct PrebuiltStringMap : PrebuiltStringMapBase { + PrebuiltStringMap(uint64_t arraySize) : PrebuiltStringMapBase(arraySize) {} + + struct ArrayElement { + StringTy key; + ElemTy value; + }; + + ArrayElement *array() { + uintptr_t start = (uintptr_t)(&arraySize + 1); + return (ArrayElement *)start; + } + + const ArrayElement *array() const { + uintptr_t start = (uintptr_t)(&arraySize + 1); + return (ArrayElement *)start; + } + + static size_t byteSize(uint64_t arraySize) { + return sizeof(PrebuiltStringMapBase) + sizeof(ArrayElement) * arraySize; + } + + /// Perform the search portion of an insertion operation. Returns a pointer to + /// the element where string is to be inserted. The caller is responsible for + /// initializing the element to contain the string/value. It is assumed that + /// the key does not already exist in the map. If it does exist, this will + /// insert a useless duplicate. + ArrayElement *insert(const void *string, size_t len) { + auto foundIndex = findIndex(string, len, [&](size_t index) { + return stringIsNull(array()[index].key); + }); + + if (foundIndex) + return &array()[*foundIndex]; + return nullptr; } ArrayElement *insert(const char *string) { @@ -154,32 +174,109 @@ struct PrebuiltStringMap { } const ArrayElement *find(const char *toFind, size_t len) const { - uint64_t hashValue = hash(toFind, len); + auto equalOrNull = [&](size_t index) { + auto key = array()[index].key; - size_t index = hashValue % arraySize; + // NULL is considered a "match" as we want to stop the search on NULL too. + if (stringIsNull(key)) + return true; - size_t numSearched = 0; - while (const char *key = array()[index].key) { // key is NUL terminated but toFind may not be. Check that they have equal // contents up to len, and check that key has a terminating NUL at the // right point. if (strncmp(key, toFind, len) == 0 && key[len] == 0) - return &array()[index]; + return true; - index = index + 1; - if (index >= arraySize) - index = 0; + // Not NULL, not equal, keep searching. + return false; + }; + auto foundIndex = findIndex(toFind, len, equalOrNull); + if (!foundIndex) + return nullptr; - numSearched++; - if (numSearched > arraySize) { - assert( - false && - "Could not find match or empty element in PrebuiltStringMap::find"); - return nullptr; - } - } + const auto &elementPtr = &array()[*foundIndex]; - return nullptr; + // If the "matching" element contains a NULL then we didn't find a match. + if (stringIsNull(elementPtr->key)) + return nullptr; + + return elementPtr; + } +}; + +/// A pre-built map with string-based keys that are implicit, i.e. equality can +/// be determined by looking at the values. The map contains auxiliary data +/// stored out of line from the main elements, to avoid padding when the aux +/// data is smaller than the alignment of the main elements. +template +struct PrebuiltAuxDataImplicitStringMap : PrebuiltStringMapBase { + PrebuiltAuxDataImplicitStringMap(uint64_t arraySize) + : PrebuiltStringMapBase(arraySize) {} + + static size_t byteSize(uint64_t arraySize) { + return sizeof(PrebuiltStringMapBase) + sizeof(ElemTy) * arraySize + + sizeof(AuxTy) * arraySize; + } + + using DataPointers = std::pair; + using DataPointersConst = std::pair; + + const ElemTy *elements() const { return (const ElemTy *)(&arraySize + 1); } + + ElemTy *elements() { return (ElemTy *)(&arraySize + 1); } + + const AuxTy *aux() const { return (const AuxTy *)(elements() + arraySize); } + + AuxTy *aux() { return (AuxTy *)(elements() + arraySize); } + + DataPointersConst pointers(size_t index) const { + return {&elements()[index], &aux()[index]}; + } + + DataPointers pointers(size_t index) { + return {&elements()[index], &aux()[index]}; + } + + /// Perform the search portion of an insertion operation. Returns pointers to + /// the element and aux data where the value is to be inserted. The caller is + /// responsible for initializing the element and aux data. It is assumed that + /// the key does not already exist in the map. If it does exist, this will + /// insert a duplicate. + /// + /// isNull is a callable passed a pair of pointers to an element and + /// corresponding auxiliary data, and must return true if the element is + /// considered NULL (empty). + template + DataPointers insert(const char *string, const IsNull &isNull) { + auto foundIndex = findIndex(string, strlen(string), [&](size_t index) { + return isNull(pointers(index)); + }); + if (!foundIndex) + return {nullptr, nullptr}; + return pointers(*foundIndex); + } + + /// Look up the given key in the map. + /// + /// isMatch is a callable passed a pair of pointers to the element and + /// auxiliary data, and must return true if the elements they point to are a + /// match for what's being looked up. + /// + /// isNull must return true if the elements are NULL/empty, as with insert(). + /// + /// The returned pointers point to the matched element and auxiliary data, if + /// a match was found. They point to a NULL entry if no map was found. They + /// will only be NULL if the table data was malformed and no match or NULL + /// exists in it. + template + DataPointersConst find(const char *toFind, size_t len, const IsMatch &isMatch, + const IsNull &isNull) const { + auto foundIndex = findIndex(toFind, len, [&](size_t index) { + return isNull(pointers(index)) || isMatch(pointers(index)); + }); + if (!foundIndex) + return {nullptr, nullptr}; + return pointers(*foundIndex); } }; diff --git a/stdlib/public/runtime/EnvironmentVariables.cpp b/stdlib/public/runtime/EnvironmentVariables.cpp index eeb9b7b0c1153..20491493e018b 100644 --- a/stdlib/public/runtime/EnvironmentVariables.cpp +++ b/stdlib/public/runtime/EnvironmentVariables.cpp @@ -161,8 +161,9 @@ void printHelp(const char *extra) { } // end anonymous namespace // Define backing variables. -#define VARIABLE(name, type, defaultValue, help) \ - type swift::runtime::environment::name ## _variable = defaultValue; +#define VARIABLE(name, type, defaultValue, help) \ + type swift::runtime::environment::name##_variable = defaultValue; \ + bool swift::runtime::environment::name##_isSet_variable = false; #include "EnvironmentVariables.def" // Initialization code. @@ -194,6 +195,11 @@ void swift::runtime::environment::initialize(void *context) { // us to detect some spelling mistakes by warning on unknown SWIFT_ variables. bool SWIFT_DEBUG_HELP_variable = false; + + // Placeholder variable, we never use the result but the macros want to write + // to it. + bool SWIFT_DEBUG_HELP_isSet_variable = false; + (void)SWIFT_DEBUG_HELP_isSet_variable; // Silence warnings about unused vars. for (char **var = ENVIRON; *var; var++) { // Immediately skip anything without a SWIFT_ prefix. if (strncmp(*var, "SWIFT_", 6) != 0) @@ -204,12 +210,13 @@ void swift::runtime::environment::initialize(void *context) { // parsed by functions named parse_ above. An unknown type will // produce an error that parse_ doesn't exist. Add new parsers // above. -#define VARIABLE(name, type, defaultValue, help) \ - if (strncmp(*var, #name "=", strlen(#name "=")) == 0) { \ - name ## _variable = \ - parse_ ## type(#name, *var + strlen(#name "="), defaultValue); \ - foundVariable = true; \ - } +#define VARIABLE(name, type, defaultValue, help) \ + if (strncmp(*var, #name "=", strlen(#name "=")) == 0) { \ + name##_variable = \ + parse_##type(#name, *var + strlen(#name "="), defaultValue); \ + name##_isSet_variable = true; \ + foundVariable = true; \ + } // SWIFT_DEBUG_HELP is not in the variables list. Parse it like the other // variables. VARIABLE(SWIFT_DEBUG_HELP, bool, false, ) @@ -238,8 +245,13 @@ void swift::runtime::environment::initialize(void *context) { void swift::runtime::environment::initialize(void *context) { // Emit a getenv call for each variable. This is less efficient but works // everywhere. -#define VARIABLE(name, type, defaultValue, help) \ - name ## _variable = parse_ ## type(#name, getenv(#name), defaultValue); +#define VARIABLE(name, type, defaultValue, help) \ + do { \ + const char name##_string = getenv(#name); \ + if (name##_string) \ + name##_isSet_variable = true; \ + name##_variable = parse_##type(#name, name##_string, defaultValue); \ + } while (0) #include "EnvironmentVariables.def" // Print help if requested. diff --git a/stdlib/public/runtime/EnvironmentVariables.def b/stdlib/public/runtime/EnvironmentVariables.def index 8b8ea3f78ad02..e384c27557b3b 100644 --- a/stdlib/public/runtime/EnvironmentVariables.def +++ b/stdlib/public/runtime/EnvironmentVariables.def @@ -78,8 +78,15 @@ VARIABLE(SWIFT_BINARY_COMPATIBILITY_VERSION, uint32_t, 0, VARIABLE(SWIFT_DEBUG_FAILED_TYPE_LOOKUP, bool, false, "Enable warnings when we fail to look up a type by name.") -VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED, bool, true, - "Enable use of prespecializations library.") +VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA, bool, true, + "Enable use of metadata in prespecializations library.") + +VARIABLE(SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP, bool, true, + "Enable use of descriptor map in prespecializations library.") + +VARIABLE(SWIFT_DEBUG_VALIDATE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP, bool, false, + "Validate results from the prespecializations map descriptor map by " + "comparing to a full scan.") VARIABLE(SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH, string, "", "A path to a prespecializations library to use at runtime. In order " diff --git a/stdlib/public/runtime/LibPrespecialized.cpp b/stdlib/public/runtime/LibPrespecialized.cpp index 7f9ff22050f4c..82e7e30999122 100644 --- a/stdlib/public/runtime/LibPrespecialized.cpp +++ b/stdlib/public/runtime/LibPrespecialized.cpp @@ -35,8 +35,6 @@ using namespace swift; -static std::atomic disablePrespecializedMetadata = false; - static bool prespecializedLoggingEnabled = false; #define LOG(fmt, ...) \ @@ -45,6 +43,8 @@ static bool prespecializedLoggingEnabled = false; fprintf(stderr, "Prespecializations library: " fmt "\n", __VA_ARGS__); \ } while (0) +#define LOG0(string) LOG("%s", string) + static bool environmentProcessListContainsProcess(const char *list, const char *progname) { auto prognameLen = strlen(progname); @@ -110,68 +110,6 @@ static bool isThisProcessEnabled(const LibPrespecializedData *data) { return true; } -static const LibPrespecializedData *findLibPrespecialized() { - if (!runtime::environment::SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED()) { - LOG("Disabling, SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED = %d", - runtime::environment::SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED()); - return nullptr; - } - - const void *dataPtr = nullptr; -#if USE_DLOPEN - auto path = runtime::environment::SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH(); - if (path && path[0]) { - // Use RTLD_NOLOAD to avoid actually loading the library. We just want to - // find it if it has already been loaded by other means, such as - // DYLD_INSERT_LIBRARIES. - void *handle = dlopen(path, RTLD_LAZY | RTLD_NOLOAD); - if (!handle) { - swift::warning(0, "Failed to load prespecializations library: %s\n", - dlerror()); - return nullptr; - } - - dataPtr = dlsym(handle, LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME); - LOG("Loaded custom library from %s, found dataPtr %p", path, dataPtr); - } -#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED - else if (SWIFT_RUNTIME_WEAK_CHECK(_dyld_get_swift_prespecialized_data)) { - // Disable the prespecializations library if anything in the shared cache is - // overridden. Eventually we want to be cleverer and only disable the - // prespecializations that have been invalidated, but we'll start with the - // simplest approach. - if (!dyld_shared_cache_some_image_overridden()) { - dataPtr = SWIFT_RUNTIME_WEAK_USE(_dyld_get_swift_prespecialized_data()); - LOG("Got dataPtr %p from _dyld_get_swift_prespecialized_data", dataPtr); - } else { - LOG("Not calling _dyld_get_swift_prespecialized_data " - "dyld_shared_cache_some_image_overridden = %d", - dyld_shared_cache_some_image_overridden()); - } - } -#endif -#endif - - if (!dataPtr) - return nullptr; - - auto *data = - reinterpret_cast *>(dataPtr); - if (data->majorVersion != - LibPrespecializedData::currentMajorVersion) { - LOG("Unknown major version %" PRIu32 ", disabling", data->majorVersion); - return nullptr; - } - - if (!isThisProcessEnabled(data)) - return nullptr; - - LOG("Returning data %p, major version %" PRIu32 " minor %" PRIu32, data, - data->majorVersion, data->minorVersion); - - return data; -} - struct LibPrespecializedState { struct AddressRange { uintptr_t start, end; @@ -182,6 +120,7 @@ struct LibPrespecializedState { }; enum class MapConfiguration { + Unset, UseNameKeyedMap, UsePointerKeyedMap, UsePointerKeyedMapDebugMode, @@ -189,9 +128,10 @@ struct LibPrespecializedState { }; const LibPrespecializedData *data; - MapConfiguration mapConfiguration; + std::atomic mapConfiguration = MapConfiguration::Unset; AddressRange sharedCacheRange{0, 0}; AddressRange metadataAllocatorInitialPoolRange{0, 0}; + bool descriptorMapEnabled; LibPrespecializedState() { prespecializedLoggingEnabled = @@ -211,8 +151,41 @@ struct LibPrespecializedState { metadataAllocatorInitialPoolRange.start + initialPoolLength; #endif - // Must do this after the shared cache range has been retrieved. - mapConfiguration = computeMapConfiguration(data); + // Compute our map configuration if it hasn't already been set. We must do + // this after the shared cache range has been retrieved, because the map + // configuration can be different depending on whether the map is in the + // shared cache. + if (mapConfiguration.load(std::memory_order_relaxed) == + MapConfiguration::Unset) + mapConfiguration.store(computeMapConfiguration(data), + std::memory_order_relaxed); + + if (data) { + descriptorMapEnabled = + data->getOptionFlags() & + LibPrespecializedData::OptionFlagDescriptorMapDefaultOn; + LOG("Setting descriptorMapEnabled=%s from the option flags.", + descriptorMapEnabled ? "true" : "false"); + } + + if (runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP_isSet()) { + descriptorMapEnabled = runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP(); + LOG("Setting descriptorMapEnabled=%s from " + "SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP.", + descriptorMapEnabled ? "true" : "false"); + } else { +#if HAS_OS_FEATURE + if (os_feature_enabled_simple(Swift, togglePrespecializationDescriptorMap, + false)) { + descriptorMapEnabled = !descriptorMapEnabled; + LOG("Toggling descriptorMapEnabled to %s " + "togglePrespecializationDescriptorMap is set.", + descriptorMapEnabled ? "true" : "false"); + } +#endif + } } MapConfiguration @@ -221,6 +194,13 @@ struct LibPrespecializedState { if (!data) return MapConfiguration::Disabled; + if (!runtime::environment:: + SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA()) { + LOG0("Disabling metadata, SWIFT_DEBUG_ENABLE_LIB_PRESPECIALIZED_METADATA " + "is false."); + return MapConfiguration::Disabled; + } + auto nameKeyedMap = data->getMetadataMap(); auto pointerKeyedMap = data->getPointerKeyedMetadataMap(); @@ -265,14 +245,75 @@ struct LibPrespecializedState { } return MapConfiguration::UseNameKeyedMap; } + + const LibPrespecializedData *findLibPrespecialized() { + const void *dataPtr = nullptr; +#if USE_DLOPEN + auto path = runtime::environment::SWIFT_DEBUG_LIB_PRESPECIALIZED_PATH(); + if (path && path[0]) { + // Use RTLD_NOLOAD to avoid actually loading the library. We just want to + // find it if it has already been loaded by other means, such as + // DYLD_INSERT_LIBRARIES. + void *handle = dlopen(path, RTLD_LAZY | RTLD_NOLOAD); + if (!handle) { + swift::warning(0, "Failed to load prespecializations library: %s\n", + dlerror()); + return nullptr; + } + + dataPtr = dlsym(handle, LIB_PRESPECIALIZED_TOP_LEVEL_SYMBOL_NAME); + LOG("Loaded custom library from %s, found dataPtr %p", path, dataPtr); + } +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + else if (SWIFT_RUNTIME_WEAK_CHECK(_dyld_get_swift_prespecialized_data)) { + dataPtr = SWIFT_RUNTIME_WEAK_USE(_dyld_get_swift_prespecialized_data()); + LOG("Got dataPtr %p from _dyld_get_swift_prespecialized_data", dataPtr); + + // Disable the prespecialized metadata if anything in the shared cache is + // overridden. Eventually we want to be cleverer and only disable the + // prespecializations that have been invalidated, but we'll start with the + // simplest approach. + if (dyld_shared_cache_some_image_overridden()) { + mapConfiguration.store(MapConfiguration::Disabled, + std::memory_order_release); + LOG("Disabling prespecialized metadata, " + "dyld_shared_cache_some_image_overridden = %d", + dyld_shared_cache_some_image_overridden()); + } + } +#endif +#endif + + LOG("Returning data pointer %p", dataPtr); + + if (!dataPtr) + return nullptr; + + auto *data = + reinterpret_cast *>(dataPtr); + if (data->majorVersion != + LibPrespecializedData::currentMajorVersion) { + LOG("Unknown major version %" PRIu32 ", disabling", data->majorVersion); + return nullptr; + } + + if (!isThisProcessEnabled(data)) + return nullptr; + + LOG("Returning data %p, major version %" PRIu32 " minor %" PRIu32, data, + data->majorVersion, data->minorVersion); + LOG(" optionFlags=%#zx", data->getOptionFlags()); + LOG(" metadataMap=%p", data->getMetadataMap()); + LOG(" disabledProcessTable=%p", data->getDisabledProcessesTable()); + LOG(" pointerKeyedMetadataMap=%p", data->getPointerKeyedMetadataMap()); + LOG(" descriptorMap=%p", data->getDescriptorMap()); + + return data; + } }; static Lazy LibPrespecialized; -const LibPrespecializedData *swift::getLibPrespecializedData() { - return SWIFT_LAZY_CONSTANT(findLibPrespecialized()); -} - // Returns true if the type has any arguments that aren't plain types (packs or // unknown kinds). static bool @@ -310,17 +351,27 @@ isPotentialPrespecializedPointer(const LibPrespecializedState &state, return true; } -static bool disableForValidation = false; +static bool isDescriptorLoaded(const void *descriptor, uint16_t imageIndex) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + return _dyld_is_preoptimized_objc_image_loaded(imageIndex); +#else + // If we're not using the dyld SPI, then we're working with a test dylib, and + // a test dylib can't have pointers to unloaded dylibs. + return true; +#endif +} void swift::libPrespecializedImageLoaded() { - #if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED // A newly loaded image might have caused us to load images that are // overriding images in the shared cache. If we do that, turn off // prespecialized metadata. if (dyld_shared_cache_some_image_overridden()) - disablePrespecializedMetadata.store(true, std::memory_order_release); - #endif + LibPrespecialized.get().mapConfiguration.store( + LibPrespecializedState::MapConfiguration::Disabled, + std::memory_order_release); +#endif } static Metadata * @@ -476,13 +527,13 @@ static Metadata *getMetadataFromPointerKeyedMapDebugMode( Metadata * swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description, const void *const *arguments) { - if (SWIFT_UNLIKELY(disableForValidation || disablePrespecializedMetadata.load( - std::memory_order_acquire))) - return nullptr; - auto &state = LibPrespecialized.get(); switch (state.mapConfiguration) { + case LibPrespecializedState::MapConfiguration::Unset: + assert(false && + "Map configuration should never be unset after initialization."); + return nullptr; case LibPrespecializedState::MapConfiguration::Disabled: return nullptr; case LibPrespecializedState::MapConfiguration::UseNameKeyedMap: @@ -495,13 +546,125 @@ swift::getLibPrespecializedMetadata(const TypeContextDescriptor *description, } } +std::pair +swift::getLibPrespecializedTypeDescriptor(Demangle::NodePointer node) { + auto &state = LibPrespecialized.get(); + + // Retrieve the map and return immediately if we don't have it. + auto *data = state.data; + if (!data) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + if (!state.descriptorMapEnabled) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + auto *descriptorMap = data->getDescriptorMap(); + if (!descriptorMap) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + + // Demangler and resolver for subsequent mangling operations. + StackAllocatedDemangler<4096> dem; + ExpandResolvedSymbolicReferences resolver{dem}; + + if (SWIFT_UNLIKELY(prespecializedLoggingEnabled)) { + auto mangling = Demangle::mangleNode(node, resolver, dem); + if (!mangling.isSuccess()) { + LOG("Failed to build demangling for node %p.", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto mangled = mangling.result(); + LOG("Looking up descriptor named '%.*s'.", (int)mangled.size(), + mangled.data()); + } + + // Get the simplified mangling that we use as the map's key. + auto simplifiedNode = buildSimplifiedDescriptorDemangling(node, dem); + if (!simplifiedNode) { + LOG("Failed to build simplified mangling for node %p.", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto simplifiedMangling = Demangle::mangleNode(simplifiedNode, resolver, dem); + if (!simplifiedMangling.isSuccess()) { + LOG("Failed to build demangling for simplified node %p.\n", node); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + // The map key is the simplified mangled name. + auto key = simplifiedMangling.result(); + + // Track how many descriptors we checked and how many were actually loaded, + // for logging. + unsigned numDescriptorsChecked = 0; + unsigned numDescriptorsLoaded = 0; + + // A descriptor is a match if it's actually loaded, and if it matches the node + // we're looking up. + auto isMatch = [&](auto pointers) { + auto *descriptor = *pointers.first; + uint16_t libraryIndex = *pointers.second; + + numDescriptorsChecked++; + + if (!isDescriptorLoaded(descriptor, libraryIndex)) + return false; + + numDescriptorsLoaded++; + + return _contextDescriptorMatchesMangling( + (const TypeContextDescriptor *)descriptor, node); + }; + + // Perform the lookup. + auto isNull = [](auto pointers) { return *pointers.first == nullptr; }; + auto found = descriptorMap->find(key.data(), key.size(), isMatch, isNull); + + LOG("Hash table lookup checked %u loaded entries, %u total entries.", + numDescriptorsLoaded, numDescriptorsChecked); + + // The pointers in `found` are pointers to the map entries, and should always + // be non-NULL. The only condition that returns NULL is if the map has no + // entries where `isMatch` or `isNull` return true, and the map should always + // have at least one NULL entry. + assert(found.first); + if (!found.first) { + LOG("Descriptor table lookup of '%.*s' returned NULL pointer to descriptor " + "pointer.", + (int)key.size(), key.data()); + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + } + + auto *foundDescriptor = *found.first; + + if (!foundDescriptor) { + LOG("Did not find descriptor for key '%.*s'.", (int)key.size(), key.data()); + + // This result is definitive if the descriptor map is comprehensive. If the + // map is not comprehensive, return NonDefinitiveNotFound to tell the caller + // that it needs to perform a full search. + if (data->getOptionFlags() & + LibPrespecializedData< + InProcess>::OptionFlagDescriptorMapNotComprehensive) + return {LibPrespecializedLookupResult::NonDefinitiveNotFound, nullptr}; + return {LibPrespecializedLookupResult::DefinitiveNotFound, nullptr}; + } + + LOG("Found descriptor %p for key '%.*s'.", foundDescriptor, (int)key.size(), + key.data()); + return {LibPrespecializedLookupResult::Found, + (const TypeContextDescriptor *)foundDescriptor}; +} + void _swift_validatePrespecializedMetadata() { - auto *data = getLibPrespecializedData(); + auto *data = LibPrespecialized.get().data; if (!data) { return; } - disableForValidation = true; + LibPrespecialized.get().mapConfiguration.store( + LibPrespecializedState::MapConfiguration::Disabled, + std::memory_order_release); unsigned validated = 0; unsigned failed = 0; diff --git a/stdlib/public/runtime/MetadataLookup.cpp b/stdlib/public/runtime/MetadataLookup.cpp index 8c252da96caa7..a199b4663534e 100644 --- a/stdlib/public/runtime/MetadataLookup.cpp +++ b/stdlib/public/runtime/MetadataLookup.cpp @@ -57,6 +57,10 @@ using namespace reflection; #include #endif +#if __has_include() +#include +#endif + /// A Demangler suitable for resolving runtime type metadata strings. template class DemanglerForRuntimeTypeResolution : public Base { @@ -325,14 +329,38 @@ namespace { }; } // end anonymous namespace +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED +struct SharedCacheInfoState { + uintptr_t dyldSharedCacheStart; + uintptr_t dyldSharedCacheEnd; + + bool inSharedCache(const void *ptr) { + auto uintPtr = reinterpret_cast(ptr); + return dyldSharedCacheStart <= uintPtr && uintPtr < dyldSharedCacheEnd; + } + + SharedCacheInfoState() { + size_t length; + dyldSharedCacheStart = (uintptr_t)_dyld_get_shared_cache_range(&length); + dyldSharedCacheEnd = + dyldSharedCacheStart ? dyldSharedCacheStart + length : 0; + } +}; + +static Lazy SharedCacheInfo; +#endif + struct TypeMetadataPrivateState { ConcurrentReadableHashMap NominalCache; ConcurrentReadableArray SectionsToScan; - + +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + ConcurrentReadableArray SharedCacheSectionsToScan; +#endif + TypeMetadataPrivateState() { initializeTypeMetadataRecordLookup(); } - }; static Lazy TypeMetadataRecords; @@ -341,6 +369,12 @@ static void _registerTypeMetadataRecords(TypeMetadataPrivateState &T, const TypeMetadataRecord *begin, const TypeMetadataRecord *end) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + if (SharedCacheInfo.get().inSharedCache(begin)) { + T.SharedCacheSectionsToScan.push_back(TypeMetadataSection{begin, end}); + return; + } +#endif T.SectionsToScan.push_back(TypeMetadataSection{begin, end}); } @@ -775,6 +809,111 @@ swift::_contextDescriptorMatchesMangling(const ContextDescriptor *context, return true; } +// Helper functions to allow _searchTypeMetadataRecordsInSections to work with +// both type and protocol records. +static const ContextDescriptor * +getContextDescriptor(const TypeMetadataRecord &record) { + return record.getContextDescriptor(); +} + +static const ContextDescriptor * +getContextDescriptor(const ProtocolRecord &record) { + return record.Protocol.getPointer(); +} + +// Perform a linear scan of the given records section, searching for a +// descriptor that matches the mangling passed in `node`. `sectionsToScan` is a +// ConcurrentReadableHashMap containing sections of type/protocol records. +template +static const ContextDescriptor * +_searchTypeMetadataRecordsInSections(SectionsContainer §ionsToScan, + Demangle::NodePointer node) { + for (auto §ion : sectionsToScan.snapshot()) { + for (const auto &record : section) { + if (auto context = getContextDescriptor(record)) { + if (_contextDescriptorMatchesMangling(context, node)) { + return context; + } + } + } + } + + return nullptr; +} + +// Search for a context descriptor matching the mangling passed in `node`. +// `state` is `TypeMetadataPrivateState` or `ProtocolMetadataPrivateState` and +// the search will use the sections in those structures. `traceBegin` is a +// function returning a trace state which is called around the linear scans of +// type/protocol records. When available, the search will consult the +// LibPrespecialized table, and perform validation on the result when validation +// is enabled. +template +static const ContextDescriptor * +_searchForContextDescriptor(State &state, NodePointer node, + TraceBegin traceBegin) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + // Try LibPrespecialized first. + auto result = getLibPrespecializedTypeDescriptor(node); + + // Validate the result if requested. + if (SWIFT_UNLIKELY( + runtime::environment:: + SWIFT_DEBUG_VALIDATE_LIB_PRESPECIALIZED_DESCRIPTOR_LOOKUP())) { + // Only validate a definitive result. + if (result.first == LibPrespecializedLookupResult::Found || + result.first == LibPrespecializedLookupResult::DefinitiveNotFound) { + // Perform a scan of the shared cache sections and see if the result + // matches. + auto scanResult = _searchTypeMetadataRecordsInSections( + state.SharedCacheSectionsToScan, node); + + // Ignore a result that's outside the shared cache. This can happen for + // indirect descriptor records that get fixed up to point to a root. + if (SharedCacheInfo.get().inSharedCache(scanResult)) { + // We may find a different but equivalent context if they're not unique, + // as iteration order may be different between the two. Use + // equalContexts to compare distinct but equal non-unique contexts + // properly. + if (!equalContexts(result.second, scanResult)) { + auto tree = getNodeTreeAsString(node); + swift::fatalError( + 0, + "Searching for type descriptor, prespecialized descriptor map " + "returned %p, but scan returned %p. Node tree:\n%s", + result.second, scanResult, tree.c_str()); + } + } + } + } + + // If we found something, we're done, return it. + if (result.first == LibPrespecializedLookupResult::Found) { + assert(result.second); + return result.second; + } + + // If a negative result was not definitive, then we must search the shared + // cache sections. + if (result.first == LibPrespecializedLookupResult::NonDefinitiveNotFound) { + auto traceState = traceBegin(node); + auto descriptor = _searchTypeMetadataRecordsInSections( + state.SharedCacheSectionsToScan, node); + traceState.end(descriptor); + if (descriptor) + return descriptor; + } + + // If we didn't find anything in the shared cache, then search the rest. +#endif + + auto traceState = traceBegin(node); + auto foundDescriptor = + _searchTypeMetadataRecordsInSections(state.SectionsToScan, node); + traceState.end(foundDescriptor); + return foundDescriptor; +} + // returns the nominal type descriptor for the type named by typeName static const ContextDescriptor * _searchTypeMetadataRecords(TypeMetadataPrivateState &T, @@ -789,19 +928,8 @@ _searchTypeMetadataRecords(TypeMetadataPrivateState &T, return nullptr; #endif - auto traceState = runtime::trace::metadata_scan_begin(node); - - for (auto §ion : T.SectionsToScan.snapshot()) { - for (const auto &record : section) { - if (auto context = record.getContextDescriptor()) { - if (_contextDescriptorMatchesMangling(context, node)) { - return traceState.end(context); - } - } - } - } - - return nullptr; + return _searchForContextDescriptor(T, node, + runtime::trace::metadata_scan_begin); } #define DESCRIPTOR_MANGLING_SUFFIX_Structure Mn @@ -980,6 +1108,10 @@ namespace { ConcurrentReadableHashMap ProtocolCache; ConcurrentReadableArray SectionsToScan; +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + ConcurrentReadableArray SharedCacheSectionsToScan; +#endif + ProtocolMetadataPrivateState() { initializeProtocolLookup(); } @@ -992,6 +1124,12 @@ static void _registerProtocols(ProtocolMetadataPrivateState &C, const ProtocolRecord *begin, const ProtocolRecord *end) { +#if DYLD_GET_SWIFT_PRESPECIALIZED_DATA_DEFINED + if (SharedCacheInfo.get().inSharedCache(begin)) { + C.SharedCacheSectionsToScan.push_back(ProtocolSection{begin, end}); + return; + } +#endif C.SectionsToScan.push_back(ProtocolSection{begin, end}); } @@ -1029,18 +1167,12 @@ void swift::swift_registerProtocols(const ProtocolRecord *begin, static const ProtocolDescriptor * _searchProtocolRecords(ProtocolMetadataPrivateState &C, NodePointer node) { - auto traceState = runtime::trace::protocol_scan_begin(node); - - for (auto §ion : C.SectionsToScan.snapshot()) { - for (const auto &record : section) { - if (auto protocol = record.Protocol.getPointer()) { - if (_contextDescriptorMatchesMangling(protocol, node)) - return traceState.end(protocol); - } - } - } - - return nullptr; + auto descriptor = + _searchForContextDescriptor(C, node, runtime::trace::protocol_scan_begin); + assert(!descriptor || + isa(descriptor) && + "Protocol record search found non-protocol descriptor."); + return reinterpret_cast(descriptor); } static const ProtocolDescriptor * diff --git a/unittests/runtime/PrebuiltStringMap.cpp b/unittests/runtime/PrebuiltStringMap.cpp index fb95a207e0fdb..28f4ad7b7d105 100644 --- a/unittests/runtime/PrebuiltStringMap.cpp +++ b/unittests/runtime/PrebuiltStringMap.cpp @@ -15,7 +15,7 @@ static bool stringIsNull(const char *str) { return str == nullptr; } -TEST(PrebuiltStringMapTest, basic) { +TEST(PrebuiltStringMapTest, PrebuiltStringMap) { auto testOnce = [&](unsigned testEntryCount) { std::vector> testVector; testVector.reserve(testEntryCount); @@ -32,6 +32,7 @@ TEST(PrebuiltStringMapTest, basic) { void *mapAllocation = calloc(1, Map::byteSize(mapSize)); Map *map = new (mapAllocation) Map(mapSize); + // Populate the map. for (auto &[key, value] : testVector) { const char *keyCStr = key.c_str(); auto *element = map->insert(keyCStr); @@ -41,6 +42,7 @@ TEST(PrebuiltStringMapTest, basic) { element->value = value; } + // Verify that we can find all the test values. for (auto &[key, value] : testVector) { const char *keyCStr = key.c_str(); auto *element = map->find(keyCStr); @@ -63,6 +65,11 @@ TEST(PrebuiltStringMapTest, basic) { EXPECT_EQ(element->value, value); } + // Verify that nonexistent keys are not found. + const char *nonexistentKey = "ceci n'est pas une clef"; + auto *element = map->find(nonexistentKey); + EXPECT_EQ(element, nullptr); + free(mapAllocation); }; @@ -70,3 +77,67 @@ TEST(PrebuiltStringMapTest, basic) { testOnce(100); testOnce(1000); } + +TEST(PrebuiltStringMapTest, PrebuiltAuxDataImplicitStringMap) { + auto testOnce = [&](unsigned testEntryCount) { + // Test a map containing positive integers, where the key is a string + // derived from the integer. The aux data is the negative of the integer. + using Map = swift::PrebuiltAuxDataImplicitStringMap; + + auto getKey = [](unsigned n) { + std::string key; + for (unsigned i = 0; i < n; i++) { + key += 'A' + (i % 26); + } + return key; + }; + + unsigned mapSize = testEntryCount * 4 / 3; + void *mapAllocation = calloc(1, Map::byteSize(mapSize)); + Map *map = new (mapAllocation) Map(mapSize); + + // Populate the map. + for (unsigned n = 0; n < testEntryCount; n++) { + auto key = getKey(n); + + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = map->insert(key.c_str(), isNull); + EXPECT_NE(pointers.first, nullptr); + EXPECT_NE(pointers.second, nullptr); + + *pointers.first = n; + *pointers.second = -(int64_t)n; + } + + // Verify that we can find all the test values. + for (unsigned n = 0; n < testEntryCount; n++) { + auto key = getKey(n); + auto keyLength = key.size(); + + // Add some trash to the end to make sure the lookup doesn't look beyond + // the specified length. + key += "xyz"; + + auto isMatch = [n](auto pointers) { return *pointers.first == n; }; + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = map->find(key.c_str(), keyLength, isMatch, isNull); + EXPECT_NE(pointers.first, nullptr); + EXPECT_NE(pointers.second, nullptr); + EXPECT_EQ(*pointers.first, n); + EXPECT_EQ(*pointers.second, -(int64_t)n); + } + + // Verfy a nonexistent value is not found. + const char *nonexistentKey = "ceci n'est pas une clef"; + auto isMatch = [](auto pointers) { return false; }; + auto isNull = [](auto pointers) { return *pointers.first == 0; }; + auto pointers = + map->find(nonexistentKey, strlen(nonexistentKey), isMatch, isNull); + EXPECT_EQ(*pointers.first, 0); + EXPECT_EQ(*pointers.second, 0); + }; + + testOnce(10); + testOnce(100); + testOnce(1000); +}