From 3c31a068865d48eef504754d0044f1ec01a6d2ff Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 17 Jan 2023 09:42:16 -0600 Subject: [PATCH 1/2] Encode link_id in tagged linkage On 64-bit, we have enough space to encode (1) the tag, (2) the `depmods` index, and (3) the offset all in a single 64-bit pointer field. This means we don't need the external `link_id` arrays, which reduces the size of many pkgimages by ~5%. On 32-bit, we don't have enough bits to implement this strategy. However, most linkages seem to be against the sysimage, and so by giving that a separate tag we can achieve similar compression because the `link_id` lists will be much shorter. --- src/staticdata.c | 117 ++++++++++++++++++++++++++++++++--------- src/staticdata_utils.c | 53 +++++++++++++++++++ 2 files changed, 145 insertions(+), 25 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index 51ef3d50f767f..4a51501dec3f3 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -357,6 +357,12 @@ typedef struct { arraylist_t fixup_types; // a list of locations of types requiring (re)caching arraylist_t fixup_objs; // a list of locations of objects requiring (re)caching arraylist_t ccallable_list; // @ccallable entry points to install +#ifdef _P64 + // On 64-bit, we can encode the "`depmods` index" in the reftag, so all we need is a mapping between + // the buildid_idx & depmods_idx: + jl_array_t *buildid_depmods_idxs; + // On 64bit, all the `link_ids_*` arrays below will be NULL. +#endif // record of build_ids for all external linkages, in order of serialization for the current sysimg/pkgimg // conceptually, the base pointer for the jth externally-linked item is determined from // i = findfirst(==(link_ids[j]), jl_build_ids) @@ -387,7 +393,8 @@ enum RefTags { SymbolRef, // symbols FunctionRef, // generic functions BuiltinFunctionRef, // builtin functions - ExternalLinkage // items defined externally (used when serializing packages) + SysimageLinkage, // reference to the sysimage (from pkgimage) + ExternalLinkage // reference to some other pkgimage }; // calling conventions for internal entry points. @@ -405,9 +412,11 @@ typedef enum { #ifdef _P64 #define RELOC_TAG_OFFSET 61 +#define DEPS_IDX_OFFSET 40 // only on 64-bit can we encode the dependency-index as part of the tagged reloc #else // this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. #define RELOC_TAG_OFFSET 29 +#define DEPS_IDX_OFFSET RELOC_TAG_OFFSET #endif #if RELOC_TAG_OFFSET <= 32 @@ -759,7 +768,7 @@ done_fields: ; } arraylist_push(&serialization_queue, (void*) v); size_t idx = serialization_queue.len - 1; - assert(serialization_queue.len < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many items to serialize"); + assert(serialization_queue.len < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "too many items to serialize"); *bp = (void*)((char*)HT_NOTFOUND + 1 + idx); } @@ -864,21 +873,28 @@ static void write_pointer(ios_t *s) JL_NOTSAFEPOINT static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) { size_t i = external_blob_index(v); if (i < n_linkage_blobs()) { + // We found the sysimg/pkg that this item links against + // Compute the relocation code + size_t offset = (uintptr_t)v - (uintptr_t)jl_linkage_blobs.items[2*i]; + offset /= sizeof(void*); + assert(offset < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to external image too large"); + + if (i == 0) + return ((uintptr_t)SysimageLinkage << RELOC_TAG_OFFSET) + offset; // sysimage assert(link_ids && jl_is_array(link_ids)); +#ifdef _P64 + uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); + return ((uintptr_t)ExternalLinkage << RELOC_TAG_OFFSET) + + (((uintptr_t)link_id_data[i]) << DEPS_IDX_OFFSET) + offset; // on 64-bit, link_ids stores the mapping from build_id_idx -> deps_idx +#else + // On 32bit, we store the image key in `link_ids` assert(jl_build_ids && jl_is_array(jl_build_ids)); uint64_t *build_id_data = (uint64_t*)jl_array_data(jl_build_ids); - // We found the sysimg/pkg that this item links against - // Store the image key in `link_ids` jl_array_grow_end(link_ids, 1); - uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); + uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); // wait until after the `grow` link_id_data[jl_array_len(link_ids)-1] = build_id_data[i]; - // Compute the relocation code - size_t offset = (uintptr_t)v - (uintptr_t)jl_linkage_blobs.items[2*i]; - offset /= sizeof(void*); - assert(offset < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to external image too large"); - // jl_printf(JL_STDOUT, "External link %ld against blob %d with key %ld at position 0x%lx with offset 0x%lx to \n", jl_array_len(link_ids), i, build_id_data[i>>1], ios_pos(s->s), offset); - // jl_(v); return ((uintptr_t)ExternalLinkage << RELOC_TAG_OFFSET) + offset; +#endif } return 0; } @@ -886,7 +902,11 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // Return the integer `id` for `v`. Generically this is looked up in `serialization_order`, // but symbols, small integers, and a couple of special items (`nothing` and the root Task) // have special handling. +#ifdef _P64 +#define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), s->buildid_depmods_idxs) +#else #define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), link_ids) +#endif static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_NOTSAFEPOINT { assert(v != NULL && "cannot get backref to NULL object"); @@ -899,7 +919,7 @@ static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t * write_uint32(s->symbols, l); ios_write(s->symbols, jl_symbol_name((jl_sym_t*)v), l + 1); size_t offset = ++nsym_tag; - assert(offset < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many symbols"); + assert(offset < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "too many symbols"); idx = (void*)((char*)HT_NOTFOUND + ((uintptr_t)SymbolRef << RELOC_TAG_OFFSET) + offset); *pidx = idx; } @@ -1144,7 +1164,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // write data and relocations newa->data = NULL; // relocation offset data /= sizeof(void*); - assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); + assert(data < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to constant data too large"); arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target if (jl_is_cpointer_type(et)) { @@ -1247,7 +1267,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 8); write_padding(s->const_data, data - ios_pos(s->const_data)); data /= sizeof(void*); - assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); + assert(data < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to constant data too large"); arraylist_push(&s->relocs_list, (void*)(reloc_offset + 8)); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target void *pdata = jl_unbox_voidpointer(jl_get_nth_field(v, 2)); @@ -1534,6 +1554,8 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) case FunctionRef: assert(offset < JL_API_MAX && "unknown function pointer id"); break; + case SysimageLinkage: + break; case ExternalLinkage: break; default: @@ -1600,15 +1622,25 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas //default: assert("corrupt relocation item id"); } + case SysimageLinkage: + assert(jl_linkage_blobs.len > 0); + return (uintptr_t)jl_linkage_blobs.items[0] + offset*sizeof(void*); case ExternalLinkage: assert(link_ids); - assert(link_index); assert(jl_build_ids); uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); uint64_t *build_id_data = (uint64_t*)jl_array_data(jl_build_ids); +#ifdef _P64 + size_t depsidx = offset >> DEPS_IDX_OFFSET; + offset &= ((size_t)1 << DEPS_IDX_OFFSET) - 1; + assert(depsidx < jl_array_len(link_ids)); // on 64-bit, here link_ids is a build_id lookup by depmod index + uint64_t build_id = link_id_data[depsidx]; +#else + assert(link_index); assert(0 <= *link_index && *link_index < jl_array_len(link_ids)); uint64_t build_id = link_id_data[*link_index]; *link_index += 1; +#endif size_t i = 0, nids = jl_array_len(jl_build_ids); while (i < nids) { if (build_id == build_id_data[i]) @@ -1800,8 +1832,12 @@ static jl_value_t *jl_delayed_reloc(jl_serializer_state *s, uintptr_t offset) JL uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; int link_index = 0; +#ifdef _P64 + jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, size, offset, s->buildid_depmods_idxs, &link_index); +#else jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, size, offset, s->link_ids_relocs, &link_index); assert(link_index < jl_array_len(s->link_ids_relocs)); +#endif return ret; } @@ -1891,13 +1927,24 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint3 reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; int gvar_link_index = 0; int external_fns_link_index = 0; + jl_array_t *link_ids = NULL; for (i = 0; i < l; i++) { uintptr_t offset = gvars[i]; uintptr_t v = 0; if (i < external_fns_begin) { - v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &gvar_link_index); +#ifdef _P64 + link_ids = s->buildid_depmods_idxs; +#else + link_ids = s->link_ids_gvars; +#endif + v = get_item_for_reloc(s, base, size, offset, link_ids, &gvar_link_index); } else { - v = get_item_for_reloc(s, base, size, offset, s->link_ids_external_fnvars, &external_fns_link_index); +#ifdef _P64 + link_ids = s->buildid_depmods_idxs; +#else + link_ids = s->link_ids_external_fnvars; +#endif + v = get_item_for_reloc(s, base, size, offset, link_ids, &external_fns_link_index); } uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); *gv = v; @@ -2218,7 +2265,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new } // In addition to the system image (where `worklist = NULL`), this can also save incremental images with external linkage -static void jl_save_system_image_to_stream(ios_t *f, +static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *udeps, jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_specializations, jl_array_t *method_roots_list, jl_array_t *ext_targets, jl_array_t *edges) JL_GC_DISABLED @@ -2266,10 +2313,15 @@ static void jl_save_system_image_to_stream(ios_t *f, arraylist_new(&s.fixup_types, 0); arraylist_new(&s.fixup_objs, 0); arraylist_new(&s.ccallable_list, 0); +#ifdef _P64 + s.buildid_depmods_idxs = image_to_depmodidx(jl_build_ids, udeps); + s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = s.link_ids_external_fnvars = NULL; +#else s.link_ids_relocs = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, 0); +#endif htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; @@ -2368,21 +2420,21 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_write_relocations(&s); } - if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { + if (sysimg.size > ((uintptr_t)1 << DEPS_IDX_OFFSET)) { jl_printf( JL_STDERR, "ERROR: system image too large: sysimg.size is %jd but the limit is %" PRIxPTR "\n", (intmax_t)sysimg.size, - ((uintptr_t)1 << RELOC_TAG_OFFSET) + ((uintptr_t)1 << DEPS_IDX_OFFSET) ); jl_exit(1); } - if (const_data.size / sizeof(void*) > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { + if (const_data.size / sizeof(void*) > ((uintptr_t)1 << DEPS_IDX_OFFSET)) { jl_printf( JL_STDERR, "ERROR: system image too large: const_data.size is %jd but the limit is %" PRIxPTR "\n", (intmax_t)const_data.size, - ((uintptr_t)1 << RELOC_TAG_OFFSET)*sizeof(void*) + ((uintptr_t)1 << DEPS_IDX_OFFSET)*sizeof(void*) ); jl_exit(1); } @@ -2458,6 +2510,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_write_value(&s, ext_targets); jl_write_value(&s, edges); } +#ifndef _P64 write_uint32(f, jl_array_len(s.link_ids_gctags)); ios_write(f, (char*)jl_array_data(s.link_ids_gctags), jl_array_len(s.link_ids_gctags)*sizeof(uint64_t)); write_uint32(f, jl_array_len(s.link_ids_relocs)); @@ -2466,6 +2519,7 @@ static void jl_save_system_image_to_stream(ios_t *f, ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars)*sizeof(uint64_t)); write_uint32(f, jl_array_len(s.link_ids_external_fnvars)); ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars)*sizeof(uint64_t)); +#endif write_uint32(f, external_fns_begin); jl_write_arraylist(s.s, &s.ccallable_list); } @@ -2592,7 +2646,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); + jl_save_system_image_to_stream(ff, *udeps, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); if (_native_data != NULL) native_functions = NULL; // make sure we don't run any Julia code concurrently before this point @@ -2765,6 +2819,9 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl offset_ext_targets = jl_read_offset(&s); offset_edges = jl_read_offset(&s); } +#ifdef _P64 + s.buildid_depmods_idxs = depmod_to_imageidx(depmods); +#else size_t nlinks_gctags = read_uint32(f); if (nlinks_gctags > 0) { s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, nlinks_gctags); @@ -2785,6 +2842,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_external_fnvars); ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint64_t)); } +#endif uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); if (s.incremental) { @@ -2811,10 +2869,19 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl *base = image_base; s.s = &sysimg; - jl_read_reloclist(&s, s.link_ids_gctags, GC_OLD); // gctags +#ifdef _P64 + jl_array_t *link_ids = NULL; +#else + jl_array_t *link_ids = s.link_ids_gctags; +#endif + jl_read_reloclist(&s, link_ids, GC_OLD); // gctags size_t sizeof_tags = ios_pos(&relocs); (void)sizeof_tags; - jl_read_reloclist(&s, s.link_ids_relocs, 0); // general relocs +#ifdef _P64 +#else + link_ids = s.link_ids_relocs; +#endif + jl_read_reloclist(&s, link_ids, 0); // general relocs // s.link_ids_gvars will be processed in `jl_update_all_gvars` // s.link_ids_external_fns will be processed in `jl_update_all_gvars` jl_update_all_gvars(&s, image, external_fns_begin); // gvars relocs diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 8ba6136e54b0e..0ae26fcfff84a 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1231,3 +1231,56 @@ JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s, uint8_t *pkgimage, int64_t } return checksum; } + +// Returns `depmodidxs` where `udeps[depmodidxs[i]]` corresponds to `build_ids[i]` +static jl_array_t *image_to_depmodidx(jl_array_t *build_ids, jl_array_t *udeps) +{ + if (!udeps || !build_ids) + return NULL; + size_t j = 0, lbids = jl_array_len(build_ids), ldeps = jl_array_len(udeps); + uint64_t *bids = (uint64_t*)jl_array_data(build_ids); + jl_array_t *depmodidxs = jl_alloc_array_1d(jl_array_int32_type, lbids); + int32_t *dmidxs = (int32_t*)jl_array_data(depmodidxs); + for (size_t i = 0; i < lbids; i++) { + dmidxs[i] = -1; + uint64_t bid = bids[i]; + j = 0; // sad that this is of O(M*N) + while (j < ldeps) { + jl_value_t *deptuple = jl_array_ptr_ref(udeps, j); + jl_module_t *depmod = (jl_module_t*)jl_fieldref(deptuple, 0); // evaluating module + jl_module_t *depmod_top = depmod; + while (depmod_top->parent != jl_main_module && depmod_top->parent != depmod_top) + depmod_top = depmod_top->parent; + if (depmod_top == jl_base_module) { + dmidxs[i] = 0; + break; + } + if (depmod_top->build_id.lo == bid) { + dmidxs[i] = j; + break; + } + j++; + } + assert(dmidxs[i] >= 0); + } + return depmodidxs; +} + +// Returns `imageidxs` where `imageidxs[i]` is the blob corresponding to `depmods[i]` +static jl_array_t *depmod_to_imageidx(jl_array_t *depmods) +{ + if (!depmods) + return NULL; + size_t ldeps = jl_array_len(depmods); + jl_array_t *imageidxs = jl_alloc_array_1d(jl_array_int32_type, ldeps); + int32_t *imgidxs = (int32_t*)jl_array_data(imageidxs); + for (size_t i = 0; i < ldeps; i++) { + imgidxs[i] = -1; + jl_value_t *depmod = jl_array_ptr_ref(depmods, i); + assert(jl_is_module(depmod)); + size_t j = external_blob_index(depmod); + assert(j < 1<<31); + imgidxs[i] = (int32_t)j; + } + return imageidxs; +} From fdfa6aee2ab8aa08fc2d11bea4fa865c72ca36bc Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 16 Feb 2023 12:33:07 -0500 Subject: [PATCH 2/2] fix format issues --- src/codegen.cpp | 6 +- src/gc.c | 3 +- src/jl_exported_data.inc | 2 - src/julia_internal.h | 10 +- src/staticdata.c | 230 +++++++++++++++------------------------ src/staticdata_utils.c | 49 ++++----- 6 files changed, 115 insertions(+), 185 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 36d0b6c85fb12..d248d744dc5bb 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4251,7 +4251,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const bool cache_valid = ctx.use_cache; bool external = false; if (ctx.external_linkage) { - if (jl_object_in_image((jl_value_t*)codeinst)) { + if (0 && jl_object_in_image((jl_value_t*)codeinst)) { // Target is present in another pkgimage cache_valid = true; external = true; @@ -5624,7 +5624,7 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod bool cache_valid = params.cache; if (params.external_linkage) { - if (jl_object_in_image((jl_value_t*)codeinst)) { + if (0 && jl_object_in_image((jl_value_t*)codeinst)) { // Target is present in another pkgimage cache_valid = true; } @@ -8536,7 +8536,7 @@ void jl_compile_workqueue( auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); bool cache_valid = params.cache; if (params.external_linkage) { - cache_valid = jl_object_in_image((jl_value_t*)codeinst); + cache_valid = 0 && jl_object_in_image((jl_value_t*)codeinst); } // WARNING: isspecsig is protected by the codegen-lock. If that lock is removed, then the isspecsig load needs to be properly atomically sequenced with this. if (cache_valid && invoke != NULL) { diff --git a/src/gc.c b/src/gc.c index 59cd18e8bd87f..743c5704a53cb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1985,6 +1985,7 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; size_t elsize = ((jl_array_t *)ary8_parent)->elsize / sizeof(jl_value_t *); + assert(elsize > 0); // Decide whether need to chunk ary8 size_t nrefs = (ary8_end - ary8_begin) / elsize; if (nrefs > MAX_REFS_AT_ONCE) { @@ -2016,6 +2017,7 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_ jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *); + assert(elsize > 0); // Decide whether need to chunk ary16 size_t nrefs = (ary16_end - ary16_begin) / elsize; if (nrefs > MAX_REFS_AT_ONCE) { @@ -2668,7 +2670,6 @@ static void gc_mark_roots(jl_gc_markqueue_t *mq) } gc_try_claim_and_push(mq, jl_all_methods, NULL); gc_try_claim_and_push(mq, _jl_debug_method_invalidation, NULL); - gc_try_claim_and_push(mq, jl_build_ids, NULL); // constants gc_try_claim_and_push(mq, jl_emptytuple_type, NULL); gc_try_claim_and_push(mq, cmpswap_names, NULL); diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index dd38560af1414..a254fba5e2b28 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -128,8 +128,6 @@ XX(jl_voidpointer_type) \ XX(jl_void_type) \ XX(jl_weakref_type) \ - XX(jl_build_ids) \ - XX(jl_linkage_blobs) \ // Data symbols that are defined inside the public libjulia #define JL_EXPORTED_DATA_SYMBOLS(XX) \ diff --git a/src/julia_internal.h b/src/julia_internal.h index 8751281a9d174..8c041701c966d 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -313,9 +313,8 @@ extern tracer_cb jl_newmeth_tracer; void jl_call_tracer(tracer_cb callback, jl_value_t *tracee); void print_func_loc(JL_STREAM *s, jl_method_t *m); extern jl_array_t *_jl_debug_method_invalidation JL_GLOBALLY_ROOTED; -extern arraylist_t jl_linkage_blobs; // external linkage: sysimg/pkgimages -extern jl_array_t *jl_build_ids JL_GLOBALLY_ROOTED; // external linkage: corresponding build_ids -extern arraylist_t jl_image_relocs; // external linkage: sysimg/pkgimages +JL_DLLEXPORT extern arraylist_t jl_linkage_blobs; // external linkage: sysimg/pkgimages +JL_DLLEXPORT extern arraylist_t jl_image_relocs; // external linkage: sysimg/pkgimages extern JL_DLLEXPORT size_t jl_page_size; extern jl_function_t *jl_typeinf_func JL_GLOBALLY_ROOTED; @@ -951,10 +950,7 @@ static inline void jl_set_gc_and_wait(void) // Query if a Julia object is if a permalloc region (due to part of a sys- pkg-image) STATIC_INLINE size_t n_linkage_blobs(void) JL_NOTSAFEPOINT { - if (!jl_build_ids) - return 0; - assert(jl_is_array(jl_build_ids)); - return jl_array_len(jl_build_ids); + return jl_image_relocs.len; } // TODO: Makes this a binary search diff --git a/src/staticdata.c b/src/staticdata.c index 4a51501dec3f3..50a79abe25a0e 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -304,12 +304,11 @@ static arraylist_t layout_table; // cache of `position(s)` for each `id` in static arraylist_t object_worklist; // used to mimic recursion by jl_serialize_reachable // Permanent list of void* (begin, end+1) pairs of system/package images we've loaded previously -// togther with their module build_ids (used for external linkage) -// jl_linkage_blobs.items[2i:2i+1] correspond to jl_build_ids[i] (0-offset indexing) +// together with their module build_ids (used for external linkage) +// jl_linkage_blobs.items[2i:2i+1] correspond to build_ids[i] (0-offset indexing) // TODO: Keep this sorted so that we can use binary-search arraylist_t jl_linkage_blobs; arraylist_t jl_image_relocs; -jl_array_t *jl_build_ids JL_GLOBALLY_ROOTED = NULL; // hash of definitions for predefined function pointers static htable_t fptr_to_id; @@ -357,15 +356,11 @@ typedef struct { arraylist_t fixup_types; // a list of locations of types requiring (re)caching arraylist_t fixup_objs; // a list of locations of objects requiring (re)caching arraylist_t ccallable_list; // @ccallable entry points to install -#ifdef _P64 - // On 64-bit, we can encode the "`depmods` index" in the reftag, so all we need is a mapping between - // the buildid_idx & depmods_idx: + // mapping from a buildid_idx to a depmods_idx jl_array_t *buildid_depmods_idxs; - // On 64bit, all the `link_ids_*` arrays below will be NULL. -#endif // record of build_ids for all external linkages, in order of serialization for the current sysimg/pkgimg // conceptually, the base pointer for the jth externally-linked item is determined from - // i = findfirst(==(link_ids[j]), jl_build_ids) + // i = findfirst(==(link_ids[j]), build_ids) // blob_base = jl_linkage_blobs.items[2i] # 0-offset indexing // We need separate lists since they are intermingled at creation but split when written. jl_array_t *link_ids_relocs; @@ -384,6 +379,16 @@ static jl_value_t *jl_bigint_type = NULL; static int gmp_limb_size = 0; static jl_sym_t *jl_docmeta_sym = NULL; +#ifdef _P64 +#define RELOC_TAG_OFFSET 61 +#define DEPS_IDX_OFFSET 40 // only on 64-bit can we encode the dependency-index as part of the tagged reloc +#else +// this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. +#define RELOC_TAG_OFFSET 29 +#define DEPS_IDX_OFFSET RELOC_TAG_OFFSET +#endif + + // Tags of category `t` are located at offsets `t << RELOC_TAG_OFFSET` // Consequently there is room for 2^RELOC_TAG_OFFSET pointers, etc enum RefTags { @@ -391,8 +396,7 @@ enum RefTags { ConstDataRef, // constant data (e.g., layouts) TagRef, // items serialized via their tags SymbolRef, // symbols - FunctionRef, // generic functions - BuiltinFunctionRef, // builtin functions + FunctionRef, // functions SysimageLinkage, // reference to the sysimage (from pkgimage) ExternalLinkage // reference to some other pkgimage }; @@ -409,15 +413,9 @@ typedef enum { JL_API_MAX } jl_callingconv_t; +// Sub-divisions of some RefTags +const uintptr_t BuiltinFunctionTag = ((uintptr_t)1 << (RELOC_TAG_OFFSET - 1)); -#ifdef _P64 -#define RELOC_TAG_OFFSET 61 -#define DEPS_IDX_OFFSET 40 // only on 64-bit can we encode the dependency-index as part of the tagged reloc -#else -// this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. -#define RELOC_TAG_OFFSET 29 -#define DEPS_IDX_OFFSET RELOC_TAG_OFFSET -#endif #if RELOC_TAG_OFFSET <= 32 typedef uint32_t reloc_t; @@ -768,7 +766,7 @@ done_fields: ; } arraylist_push(&serialization_queue, (void*) v); size_t idx = serialization_queue.len - 1; - assert(serialization_queue.len < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "too many items to serialize"); + assert(serialization_queue.len < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many items to serialize"); *bp = (void*)((char*)HT_NOTFOUND + 1 + idx); } @@ -878,23 +876,18 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ size_t offset = (uintptr_t)v - (uintptr_t)jl_linkage_blobs.items[2*i]; offset /= sizeof(void*); assert(offset < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to external image too large"); - - if (i == 0) - return ((uintptr_t)SysimageLinkage << RELOC_TAG_OFFSET) + offset; // sysimage + assert(n_linkage_blobs() == jl_array_len(s->buildid_depmods_idxs)); + size_t depsidx = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[i]; // map from build_id_idx -> deps_idx + assert(depsidx < INT32_MAX); + if (depsidx < ((uintptr_t)1 << (RELOC_TAG_OFFSET - DEPS_IDX_OFFSET)) && offset < ((uintptr_t)1 << DEPS_IDX_OFFSET)) + // if it fits in a SysimageLinkage type, use that representation + return ((uintptr_t)SysimageLinkage << RELOC_TAG_OFFSET) + ((uintptr_t)depsidx << DEPS_IDX_OFFSET) + offset; + // otherwise, we store the image key in `link_ids` assert(link_ids && jl_is_array(link_ids)); -#ifdef _P64 - uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); - return ((uintptr_t)ExternalLinkage << RELOC_TAG_OFFSET) - + (((uintptr_t)link_id_data[i]) << DEPS_IDX_OFFSET) + offset; // on 64-bit, link_ids stores the mapping from build_id_idx -> deps_idx -#else - // On 32bit, we store the image key in `link_ids` - assert(jl_build_ids && jl_is_array(jl_build_ids)); - uint64_t *build_id_data = (uint64_t*)jl_array_data(jl_build_ids); jl_array_grow_end(link_ids, 1); - uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); // wait until after the `grow` - link_id_data[jl_array_len(link_ids)-1] = build_id_data[i]; + uint32_t *link_id_data = (uint32_t*)jl_array_data(link_ids); // wait until after the `grow` + link_id_data[jl_array_len(link_ids) - 1] = depsidx; return ((uintptr_t)ExternalLinkage << RELOC_TAG_OFFSET) + offset; -#endif } return 0; } @@ -902,11 +895,7 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // Return the integer `id` for `v`. Generically this is looked up in `serialization_order`, // but symbols, small integers, and a couple of special items (`nothing` and the root Task) // have special handling. -#ifdef _P64 -#define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), s->buildid_depmods_idxs) -#else #define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), link_ids) -#endif static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_NOTSAFEPOINT { assert(v != NULL && "cannot get backref to NULL object"); @@ -919,7 +908,7 @@ static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t * write_uint32(s->symbols, l); ios_write(s->symbols, jl_symbol_name((jl_sym_t*)v), l + 1); size_t offset = ++nsym_tag; - assert(offset < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "too many symbols"); + assert(offset < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "too many symbols"); idx = (void*)((char*)HT_NOTFOUND + ((uintptr_t)SymbolRef << RELOC_TAG_OFFSET) + offset); *pidx = idx; } @@ -1164,7 +1153,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // write data and relocations newa->data = NULL; // relocation offset data /= sizeof(void*); - assert(data < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to constant data too large"); + assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target if (jl_is_cpointer_type(et)) { @@ -1267,7 +1256,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), 8); write_padding(s->const_data, data - ios_pos(s->const_data)); data /= sizeof(void*); - assert(data < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to constant data too large"); + assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); arraylist_push(&s->relocs_list, (void*)(reloc_offset + 8)); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target void *pdata = jl_unbox_voidpointer(jl_get_nth_field(v, 2)); @@ -1419,12 +1408,13 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } newm->invoke = NULL; // relocation offset if (fptr_id != JL_API_NULL) { + assert(fptr_id < BuiltinFunctionTag && "too many functions to serialize"); arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, invoke))); // relocation location arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + fptr_id)); // relocation target } if (builtin_id >= 2) { arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_code_instance_t, specptr.fptr))); // relocation location - arraylist_push(&s->relocs_list, (void*)(((uintptr_t)BuiltinFunctionRef << RELOC_TAG_OFFSET) + builtin_id - 2)); // relocation target + arraylist_push(&s->relocs_list, (void*)(((uintptr_t)FunctionRef << RELOC_TAG_OFFSET) + BuiltinFunctionTag + builtin_id - 2)); // relocation target } } else if (jl_is_datatype(v)) { @@ -1548,11 +1538,14 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) case TagRef: assert(offset < 2 * NBOX_C + 258 && "corrupt relocation item id"); break; - case BuiltinFunctionRef: - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id"); - break; case FunctionRef: - assert(offset < JL_API_MAX && "unknown function pointer id"); + if (offset & BuiltinFunctionTag) { + offset &= ~BuiltinFunctionTag; + assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id"); + } + else { + assert(offset < JL_API_MAX && "unknown function pointer id"); + } break; case SysimageLinkage: break; @@ -1598,10 +1591,12 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas // offset -= 256; assert(0 && "corrupt relocation item id"); jl_unreachable(); // terminate control flow if assertion is disabled. - case BuiltinFunctionRef: - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID"); - return (uintptr_t)id_to_fptrs[offset]; case FunctionRef: + if (offset & BuiltinFunctionTag) { + offset &= ~BuiltinFunctionTag; + assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID"); + return (uintptr_t)id_to_fptrs[offset]; + } switch ((jl_callingconv_t)offset) { case JL_API_BOXED: if (s->image->fptrs.base) @@ -1622,35 +1617,30 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas //default: assert("corrupt relocation item id"); } - case SysimageLinkage: - assert(jl_linkage_blobs.len > 0); - return (uintptr_t)jl_linkage_blobs.items[0] + offset*sizeof(void*); - case ExternalLinkage: - assert(link_ids); - assert(jl_build_ids); - uint64_t *link_id_data = (uint64_t*)jl_array_data(link_ids); - uint64_t *build_id_data = (uint64_t*)jl_array_data(jl_build_ids); + case SysimageLinkage: { #ifdef _P64 size_t depsidx = offset >> DEPS_IDX_OFFSET; offset &= ((size_t)1 << DEPS_IDX_OFFSET) - 1; - assert(depsidx < jl_array_len(link_ids)); // on 64-bit, here link_ids is a build_id lookup by depmod index - uint64_t build_id = link_id_data[depsidx]; #else + size_t depsidx = 0; +#endif + assert(depsidx < jl_array_len(s->buildid_depmods_idxs)); + size_t i = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[depsidx]; + assert(2*i < jl_linkage_blobs.len); + return (uintptr_t)jl_linkage_blobs.items[2*i] + offset*sizeof(void*); + } + case ExternalLinkage: { + assert(link_ids); assert(link_index); assert(0 <= *link_index && *link_index < jl_array_len(link_ids)); - uint64_t build_id = link_id_data[*link_index]; + uint32_t depsidx = ((uint32_t*)jl_array_data(link_ids))[*link_index]; *link_index += 1; -#endif - size_t i = 0, nids = jl_array_len(jl_build_ids); - while (i < nids) { - if (build_id == build_id_data[i]) - break; - i++; - } - assert(i < nids); + assert(depsidx < jl_array_len(s->buildid_depmods_idxs)); + size_t i = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[depsidx]; assert(2*i < jl_linkage_blobs.len); return (uintptr_t)jl_linkage_blobs.items[2*i] + offset*sizeof(void*); } + } abort(); } @@ -1832,12 +1822,8 @@ static jl_value_t *jl_delayed_reloc(jl_serializer_state *s, uintptr_t offset) JL uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; int link_index = 0; -#ifdef _P64 - jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, size, offset, s->buildid_depmods_idxs, &link_index); -#else jl_value_t *ret = (jl_value_t*)get_item_for_reloc(s, base, size, offset, s->link_ids_relocs, &link_index); - assert(link_index < jl_array_len(s->link_ids_relocs)); -#endif + assert(!s->link_ids_relocs || link_index < jl_array_len(s->link_ids_relocs)); return ret; } @@ -1927,24 +1913,14 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint3 reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; int gvar_link_index = 0; int external_fns_link_index = 0; - jl_array_t *link_ids = NULL; for (i = 0; i < l; i++) { uintptr_t offset = gvars[i]; uintptr_t v = 0; if (i < external_fns_begin) { -#ifdef _P64 - link_ids = s->buildid_depmods_idxs; -#else - link_ids = s->link_ids_gvars; -#endif - v = get_item_for_reloc(s, base, size, offset, link_ids, &gvar_link_index); - } else { -#ifdef _P64 - link_ids = s->buildid_depmods_idxs; -#else - link_ids = s->link_ids_external_fnvars; -#endif - v = get_item_for_reloc(s, base, size, offset, link_ids, &external_fns_link_index); + v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &gvar_link_index); + } + else { + v = get_item_for_reloc(s, base, size, offset, s->link_ids_external_fnvars, &external_fns_link_index); } uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); *gv = v; @@ -2265,7 +2241,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new } // In addition to the system image (where `worklist = NULL`), this can also save incremental images with external linkage -static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *udeps, +static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_specializations, jl_array_t *method_roots_list, jl_array_t *ext_targets, jl_array_t *edges) JL_GC_DISABLED @@ -2313,15 +2289,11 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *udeps, arraylist_new(&s.fixup_types, 0); arraylist_new(&s.fixup_objs, 0); arraylist_new(&s.ccallable_list, 0); -#ifdef _P64 - s.buildid_depmods_idxs = image_to_depmodidx(jl_build_ids, udeps); - s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = s.link_ids_external_fnvars = NULL; -#else - s.link_ids_relocs = jl_alloc_array_1d(jl_array_uint64_type, 0); - s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, 0); - s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, 0); - s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, 0); -#endif + s.buildid_depmods_idxs = image_to_depmodidx(mod_array); + s.link_ids_relocs = jl_alloc_array_1d(jl_array_int32_type, 0); + s.link_ids_gctags = jl_alloc_array_1d(jl_array_int32_type, 0); + s.link_ids_gvars = jl_alloc_array_1d(jl_array_int32_type, 0); + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, 0); htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; @@ -2420,21 +2392,21 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *udeps, jl_write_relocations(&s); } - if (sysimg.size > ((uintptr_t)1 << DEPS_IDX_OFFSET)) { + if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { jl_printf( JL_STDERR, "ERROR: system image too large: sysimg.size is %jd but the limit is %" PRIxPTR "\n", (intmax_t)sysimg.size, - ((uintptr_t)1 << DEPS_IDX_OFFSET) + ((uintptr_t)1 << RELOC_TAG_OFFSET) ); jl_exit(1); } - if (const_data.size / sizeof(void*) > ((uintptr_t)1 << DEPS_IDX_OFFSET)) { + if (const_data.size / sizeof(void*) > ((uintptr_t)1 << RELOC_TAG_OFFSET)) { jl_printf( JL_STDERR, "ERROR: system image too large: const_data.size is %jd but the limit is %" PRIxPTR "\n", (intmax_t)const_data.size, - ((uintptr_t)1 << DEPS_IDX_OFFSET)*sizeof(void*) + ((uintptr_t)1 << RELOC_TAG_OFFSET)*sizeof(void*) ); jl_exit(1); } @@ -2510,25 +2482,17 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *udeps, jl_write_value(&s, ext_targets); jl_write_value(&s, edges); } -#ifndef _P64 write_uint32(f, jl_array_len(s.link_ids_gctags)); - ios_write(f, (char*)jl_array_data(s.link_ids_gctags), jl_array_len(s.link_ids_gctags)*sizeof(uint64_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_gctags), jl_array_len(s.link_ids_gctags) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_relocs)); - ios_write(f, (char*)jl_array_data(s.link_ids_relocs), jl_array_len(s.link_ids_relocs)*sizeof(uint64_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_relocs), jl_array_len(s.link_ids_relocs) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_gvars)); - ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars)*sizeof(uint64_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_external_fnvars)); - ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars)*sizeof(uint64_t)); -#endif + ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars) * sizeof(uint32_t)); write_uint32(f, external_fns_begin); jl_write_arraylist(s.s, &s.ccallable_list); } - // Write the build_id key - uint64_t buildid = 0; - if (worklist) - buildid = jl_worklist_key(worklist); - write_uint32(f, buildid >> 32); - write_uint32(f, buildid & (((uint64_t)1 << 32) - 1)); assert(object_worklist.len == 0); arraylist_free(&object_worklist); @@ -2646,7 +2610,7 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, *udeps, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); + jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); if (_native_data != NULL) native_functions = NULL; // make sure we don't run any Julia code concurrently before this point @@ -2819,30 +2783,27 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl offset_ext_targets = jl_read_offset(&s); offset_edges = jl_read_offset(&s); } -#ifdef _P64 s.buildid_depmods_idxs = depmod_to_imageidx(depmods); -#else size_t nlinks_gctags = read_uint32(f); if (nlinks_gctags > 0) { - s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, nlinks_gctags); - ios_read(f, (char*)jl_array_data(s.link_ids_gctags), nlinks_gctags * sizeof(uint64_t)); + s.link_ids_gctags = jl_alloc_array_1d(jl_array_int32_type, nlinks_gctags); + ios_read(f, (char*)jl_array_data(s.link_ids_gctags), nlinks_gctags * sizeof(uint32_t)); } size_t nlinks_relocs = read_uint32(f); if (nlinks_relocs > 0) { - s.link_ids_relocs = jl_alloc_array_1d(jl_array_uint64_type, nlinks_relocs); - ios_read(f, (char*)jl_array_data(s.link_ids_relocs), nlinks_relocs * sizeof(uint64_t)); + s.link_ids_relocs = jl_alloc_array_1d(jl_array_int32_type, nlinks_relocs); + ios_read(f, (char*)jl_array_data(s.link_ids_relocs), nlinks_relocs * sizeof(uint32_t)); } size_t nlinks_gvars = read_uint32(f); if (nlinks_gvars > 0) { - s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_gvars); - ios_read(f, (char*)jl_array_data(s.link_ids_gvars), nlinks_gvars * sizeof(uint64_t)); + s.link_ids_gvars = jl_alloc_array_1d(jl_array_int32_type, nlinks_gvars); + ios_read(f, (char*)jl_array_data(s.link_ids_gvars), nlinks_gvars * sizeof(uint32_t)); } size_t nlinks_external_fnvars = read_uint32(f); if (nlinks_external_fnvars > 0) { - s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_external_fnvars); - ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint64_t)); + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, nlinks_external_fnvars); + ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint32_t)); } -#endif uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); if (s.incremental) { @@ -2869,19 +2830,10 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl *base = image_base; s.s = &sysimg; -#ifdef _P64 - jl_array_t *link_ids = NULL; -#else - jl_array_t *link_ids = s.link_ids_gctags; -#endif - jl_read_reloclist(&s, link_ids, GC_OLD); // gctags + jl_read_reloclist(&s, s.link_ids_gctags, GC_OLD); // gctags size_t sizeof_tags = ios_pos(&relocs); (void)sizeof_tags; -#ifdef _P64 -#else - link_ids = s.link_ids_relocs; -#endif - jl_read_reloclist(&s, link_ids, 0); // general relocs + jl_read_reloclist(&s, s.link_ids_relocs, 0); // general relocs // s.link_ids_gvars will be processed in `jl_update_all_gvars` // s.link_ids_external_fns will be processed in `jl_update_all_gvars` jl_update_all_gvars(&s, image, external_fns_begin); // gvars relocs @@ -3237,12 +3189,6 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl arraylist_push(&jl_image_relocs, (void*)relocs_base); // jl_printf(JL_STDOUT, "%ld blobs to link against\n", jl_linkage_blobs.len >> 1); - uint64_t buildid = (((uint64_t)read_uint32(f)) << 32) | read_uint32(f); - if (!jl_build_ids) - jl_build_ids = jl_alloc_array_1d(jl_array_uint64_type, 0); - jl_array_grow_end(jl_build_ids, 1); - uint64_t *build_id_data = (uint64_t*)jl_array_data(jl_build_ids); - build_id_data[jl_array_len(jl_build_ids)-1] = buildid; jl_gc_enable(en); } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 0ae26fcfff84a..e577394c67fae 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1232,55 +1232,44 @@ JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s, uint8_t *pkgimage, int64_t return checksum; } -// Returns `depmodidxs` where `udeps[depmodidxs[i]]` corresponds to `build_ids[i]` -static jl_array_t *image_to_depmodidx(jl_array_t *build_ids, jl_array_t *udeps) +// Returns `depmodidxs` where `j = depmodidxs[i]` corresponds to the blob `depmods[j]` in `write_mod_list` +static jl_array_t *image_to_depmodidx(jl_array_t *depmods) { - if (!udeps || !build_ids) + if (!depmods) return NULL; - size_t j = 0, lbids = jl_array_len(build_ids), ldeps = jl_array_len(udeps); - uint64_t *bids = (uint64_t*)jl_array_data(build_ids); + assert(jl_array_len(depmods) < INT32_MAX && "too many dependencies to serialize"); + size_t lbids = n_linkage_blobs(); + size_t ldeps = jl_array_len(depmods); jl_array_t *depmodidxs = jl_alloc_array_1d(jl_array_int32_type, lbids); int32_t *dmidxs = (int32_t*)jl_array_data(depmodidxs); - for (size_t i = 0; i < lbids; i++) { - dmidxs[i] = -1; - uint64_t bid = bids[i]; - j = 0; // sad that this is of O(M*N) - while (j < ldeps) { - jl_value_t *deptuple = jl_array_ptr_ref(udeps, j); - jl_module_t *depmod = (jl_module_t*)jl_fieldref(deptuple, 0); // evaluating module - jl_module_t *depmod_top = depmod; - while (depmod_top->parent != jl_main_module && depmod_top->parent != depmod_top) - depmod_top = depmod_top->parent; - if (depmod_top == jl_base_module) { - dmidxs[i] = 0; - break; - } - if (depmod_top->build_id.lo == bid) { - dmidxs[i] = j; - break; - } + memset(dmidxs, -1, lbids * sizeof(int32_t)); + dmidxs[0] = 0; // the sysimg can also be found at idx 0, by construction + for (size_t i = 0, j = 0; i < ldeps; i++) { + jl_value_t *depmod = jl_array_ptr_ref(depmods, i); + size_t idx = external_blob_index(depmod); + if (idx < lbids) { // jl_object_in_image j++; + if (dmidxs[idx] == -1) + dmidxs[idx] = j; } - assert(dmidxs[i] >= 0); } return depmodidxs; } -// Returns `imageidxs` where `imageidxs[i]` is the blob corresponding to `depmods[i]` +// Returns `imageidxs` where `j = imageidxs[i]` is the blob corresponding to `depmods[j]` static jl_array_t *depmod_to_imageidx(jl_array_t *depmods) { if (!depmods) return NULL; size_t ldeps = jl_array_len(depmods); - jl_array_t *imageidxs = jl_alloc_array_1d(jl_array_int32_type, ldeps); + jl_array_t *imageidxs = jl_alloc_array_1d(jl_array_int32_type, ldeps + 1); int32_t *imgidxs = (int32_t*)jl_array_data(imageidxs); + imgidxs[0] = 0; for (size_t i = 0; i < ldeps; i++) { - imgidxs[i] = -1; jl_value_t *depmod = jl_array_ptr_ref(depmods, i); - assert(jl_is_module(depmod)); size_t j = external_blob_index(depmod); - assert(j < 1<<31); - imgidxs[i] = (int32_t)j; + assert(j < INT32_MAX); + imgidxs[i + 1] = (int32_t)j; } return imageidxs; }