Skip to content

Commit 7e2d803

Browse files
vtjnashvtjnash
authored
add logic to prefer loading modules that are already loaded (#55908)
Iterate over the list of existing loaded modules for PkgId whenever loading a new module for PkgId, so that we will use that existing build_id content if it otherwise passes the other stale_checks.
1 parent 80d67d5 commit 7e2d803

File tree

3 files changed

+138
-84
lines changed

3 files changed

+138
-84
lines changed

‎base/Base.jl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,7 @@ function __init__()
648648
empty!(explicit_loaded_modules)
649649
empty!(loaded_precompiles) # If we load a packageimage when building the image this might not be empty
650650
for (mod, key) in module_keys
651-
loaded_precompiles[key => module_build_id(mod)] = mod
651+
push!(get!(Vector{Module}, loaded_precompiles, key), mod)
652652
end
653653
if haskey(ENV, "JULIA_MAX_NUM_PRECOMPILE_FILES")
654654
MAX_NUM_PRECOMPILE_FILES[] = parse(Int, ENV["JULIA_MAX_NUM_PRECOMPILE_FILES"])

‎base/loading.jl‎

Lines changed: 103 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1231,7 +1231,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No
12311231
dep = depmods[i]
12321232
dep isa Module && continue
12331233
_, depkey, depbuild_id = dep::Tuple{String, PkgId, UInt128}
1234-
dep = loaded_precompiles[depkey => depbuild_id]
1234+
dep = something(maybe_loaded_precompile(depkey, depbuild_id))
12351235
@assert PkgId(dep) == depkey && module_build_id(dep) === depbuild_id
12361236
depmods[i] = dep
12371237
end
@@ -1337,6 +1337,7 @@ end
13371337

13381338
function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
13391339
# This function is also used by PkgCacheInspector.jl
1340+
assert_havelock(require_lock)
13401341
restored = sv[1]::Vector{Any}
13411342
for M in restored
13421343
M = M::Module
@@ -1345,7 +1346,7 @@ function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String)
13451346
end
13461347
if parentmodule(M) === M
13471348
push!(loaded_modules_order, M)
1348-
loaded_precompiles[pkg => module_build_id(M)] = M
1349+
push!(get!(Vector{Module}, loaded_precompiles, pkg), M)
13491350
end
13501351
end
13511352

@@ -1945,90 +1946,102 @@ end
19451946
assert_havelock(require_lock)
19461947
paths = find_all_in_cache_path(pkg, DEPOT_PATH)
19471948
newdeps = PkgId[]
1948-
for path_to_try in paths::Vector{String}
1949-
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
1950-
if staledeps === true
1951-
continue
1952-
end
1953-
try
1954-
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
1955-
# finish checking staledeps module graph
1956-
for i in eachindex(staledeps)
1957-
dep = staledeps[i]
1958-
dep isa Module && continue
1959-
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
1960-
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
1961-
for modpath_to_try in modpaths
1962-
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
1963-
if modstaledeps === true
1964-
continue
1965-
end
1966-
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
1967-
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
1968-
@goto check_next_dep
1949+
try_build_ids = UInt128[build_id]
1950+
if build_id == UInt128(0)
1951+
let loaded = get(loaded_precompiles, pkg, nothing)
1952+
if loaded !== nothing
1953+
for mod in loaded # try these in reverse original load order to see if one is already valid
1954+
pushfirst!(try_build_ids, module_build_id(mod))
19691955
end
1970-
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
1971-
@goto check_next_path
1972-
@label check_next_dep
1973-
end
1974-
M = get(loaded_precompiles, pkg => newbuild_id, nothing)
1975-
if isa(M, Module)
1976-
stalecheck && register_root_module(M)
1977-
return M
19781956
end
1979-
if stalecheck
1980-
try
1981-
touch(path_to_try) # update timestamp of precompilation file
1982-
catch ex # file might be read-only and then we fail to update timestamp, which is fine
1983-
ex isa IOError || rethrow()
1984-
end
1957+
end
1958+
end
1959+
for build_id in try_build_ids
1960+
for path_to_try in paths::Vector{String}
1961+
staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try; reasons, stalecheck)
1962+
if staledeps === true
1963+
continue
19851964
end
1986-
# finish loading module graph into staledeps
1987-
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
1988-
for i in eachindex(staledeps)
1989-
dep = staledeps[i]
1990-
dep isa Module && continue
1991-
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
1992-
dep = start_loading(modkey, modbuild_id, stalecheck)
1993-
while true
1994-
if dep isa Module
1995-
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
1996-
break
1997-
else
1998-
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
1999-
@goto check_next_path
1965+
try
1966+
staledeps, ocachefile, newbuild_id = staledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
1967+
# finish checking staledeps module graph
1968+
for i in eachindex(staledeps)
1969+
dep = staledeps[i]
1970+
dep isa Module && continue
1971+
modpath, modkey, modbuild_id = dep::Tuple{String, PkgId, UInt128}
1972+
modpaths = find_all_in_cache_path(modkey, DEPOT_PATH)
1973+
for modpath_to_try in modpaths
1974+
modstaledeps = stale_cachefile(modkey, modbuild_id, modpath, modpath_to_try; stalecheck)
1975+
if modstaledeps === true
1976+
continue
20001977
end
1978+
modstaledeps, modocachepath, _ = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}, UInt128}
1979+
staledeps[i] = (modpath, modkey, modbuild_id, modpath_to_try, modstaledeps, modocachepath)
1980+
@goto check_next_dep
1981+
end
1982+
@debug "Rejecting cache file $path_to_try because required dependency $modkey with build ID $(UUID(modbuild_id)) is missing from the cache."
1983+
@goto check_next_path
1984+
@label check_next_dep
1985+
end
1986+
M = maybe_loaded_precompile(pkg, newbuild_id)
1987+
if isa(M, Module)
1988+
stalecheck && register_root_module(M)
1989+
return M
1990+
end
1991+
if stalecheck
1992+
try
1993+
touch(path_to_try) # update timestamp of precompilation file
1994+
catch ex # file might be read-only and then we fail to update timestamp, which is fine
1995+
ex isa IOError || rethrow()
20011996
end
2002-
if dep === nothing
2003-
try
2004-
set_pkgorigin_version_path(modkey, modpath)
2005-
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
2006-
finally
2007-
end_loading(modkey, dep)
1997+
end
1998+
# finish loading module graph into staledeps
1999+
# TODO: call all start_loading calls (in reverse order) before calling any _include_from_serialized, since start_loading will drop the loading lock
2000+
for i in eachindex(staledeps)
2001+
dep = staledeps[i]
2002+
dep isa Module && continue
2003+
modpath, modkey, modbuild_id, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, UInt128, String, Vector{Any}, Union{Nothing, String}}
2004+
dep = start_loading(modkey, modbuild_id, stalecheck)
2005+
while true
2006+
if dep isa Module
2007+
if PkgId(dep) == modkey && module_build_id(dep) === modbuild_id
2008+
break
2009+
else
2010+
@debug "Rejecting cache file $path_to_try because module $modkey got loaded at a different version than expected."
2011+
@goto check_next_path
2012+
end
20082013
end
2009-
if !isa(dep, Module)
2010-
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
2011-
@goto check_next_path
2012-
else
2013-
push!(newdeps, modkey)
2014+
if dep === nothing
2015+
try
2016+
set_pkgorigin_version_path(modkey, modpath)
2017+
dep = _include_from_serialized(modkey, modcachepath, modocachepath, modstaledeps; register = stalecheck)
2018+
finally
2019+
end_loading(modkey, dep)
2020+
end
2021+
if !isa(dep, Module)
2022+
@debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep
2023+
@goto check_next_path
2024+
else
2025+
push!(newdeps, modkey)
2026+
end
20142027
end
20152028
end
2029+
staledeps[i] = dep
20162030
end
2017-
staledeps[i] = dep
2018-
end
2019-
restored = get(loaded_precompiles, pkg => newbuild_id, nothing)
2020-
if !isa(restored, Module)
2021-
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
2022-
end
2023-
isa(restored, Module) && return restored
2024-
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
2025-
@label check_next_path
2026-
finally
2027-
for modkey in newdeps
2028-
insert_extension_triggers(modkey)
2029-
stalecheck && run_package_callbacks(modkey)
2031+
restored = maybe_loaded_precompile(pkg, newbuild_id)
2032+
if !isa(restored, Module)
2033+
restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps; register = stalecheck)
2034+
end
2035+
isa(restored, Module) && return restored
2036+
@debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored
2037+
@label check_next_path
2038+
finally
2039+
for modkey in newdeps
2040+
insert_extension_triggers(modkey)
2041+
stalecheck && run_package_callbacks(modkey)
2042+
end
2043+
empty!(newdeps)
20302044
end
2031-
empty!(newdeps)
20322045
end
20332046
end
20342047
return nothing
@@ -2047,7 +2060,7 @@ function start_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool)
20472060
loaded = stalecheck ? maybe_root_module(modkey) : nothing
20482061
loaded isa Module && return loaded
20492062
if build_id != UInt128(0)
2050-
loaded = get(loaded_precompiles, modkey => build_id, nothing)
2063+
loaded = maybe_loaded_precompile(modkey, build_id)
20512064
loaded isa Module && return loaded
20522065
end
20532066
loading = get(package_locks, modkey, nothing)
@@ -2377,12 +2390,21 @@ const pkgorigins = Dict{PkgId,PkgOrigin}()
23772390

23782391
const explicit_loaded_modules = Dict{PkgId,Module}() # Emptied on Julia start
23792392
const loaded_modules = Dict{PkgId,Module}() # available to be explicitly loaded
2380-
const loaded_precompiles = Dict{Pair{PkgId,UInt128},Module}() # extended (complete) list of modules, available to be loaded
2393+
const loaded_precompiles = Dict{PkgId,Vector{Module}}() # extended (complete) list of modules, available to be loaded
23812394
const loaded_modules_order = Vector{Module}()
23822395
const module_keys = IdDict{Module,PkgId}() # the reverse of loaded_modules
23832396

23842397
root_module_key(m::Module) = @lock require_lock module_keys[m]
23852398

2399+
function maybe_loaded_precompile(key::PkgId, buildid::UInt128)
2400+
assert_havelock(require_lock)
2401+
mods = get(loaded_precompiles, key, nothing)
2402+
mods === nothing && return
2403+
for mod in mods
2404+
module_build_id(mod) == buildid && return mod
2405+
end
2406+
end
2407+
23862408
function module_build_id(m::Module)
23872409
hi, lo = ccall(:jl_module_build_id, NTuple{2,UInt64}, (Any,), m)
23882410
return (UInt128(hi) << 64) | lo
@@ -2403,7 +2425,7 @@ end
24032425
end
24042426
end
24052427
end
2406-
haskey(loaded_precompiles, key => module_build_id(m)) || push!(loaded_modules_order, m)
2428+
maybe_loaded_precompile(key, module_build_id(m)) === nothing && push!(loaded_modules_order, m)
24072429
loaded_modules[key] = m
24082430
explicit_loaded_modules[key] = m
24092431
module_keys[m] = key
@@ -3789,8 +3811,8 @@ end
37893811
for i in 1:ndeps
37903812
req_key, req_build_id = required_modules[i]
37913813
# Check if module is already loaded
3792-
if !stalecheck && haskey(loaded_precompiles, req_key => req_build_id)
3793-
M = loaded_precompiles[req_key => req_build_id]
3814+
M = stalecheck ? nothing : maybe_loaded_precompile(req_key, req_build_id)
3815+
if M !== nothing
37943816
@assert PkgId(M) == req_key && module_build_id(M) === req_build_id
37953817
depmods[i] = M
37963818
elseif root_module_exists(req_key)

‎test/loading.jl‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
# This file is a part of Julia. License is MIT: https://julialang.org/license
22

3-
original_depot_path = copy(Base.DEPOT_PATH)
4-
53
using Test
64

75
# Tests for @__LINE__ inside and outside of macros
6+
# NOTE: the __LINE__ numbers for these first couple tests are significant, so
7+
# adding any lines here will make those tests fail
88
@test (@__LINE__) == 8
99

1010
macro macro_caller_lineno()
@@ -33,6 +33,9 @@ end
3333
@test @nested_LINE_expansion() == ((@__LINE__() - 4, @__LINE__() - 12), @__LINE__())
3434
@test @nested_LINE_expansion2() == ((@__LINE__() - 5, @__LINE__() - 9), @__LINE__())
3535

36+
original_depot_path = copy(Base.DEPOT_PATH)
37+
include("precompile_utils.jl")
38+
3639
loaded_files = String[]
3740
push!(Base.include_callbacks, (mod::Module, fn::String) -> push!(loaded_files, fn))
3841
include("test_sourcepath.jl")
@@ -1603,3 +1606,32 @@ end
16031606
copy!(LOAD_PATH, old_load_path)
16041607
end
16051608
end
1609+
1610+
@testset "require_stdlib loading duplication" begin
1611+
depot_path = mktempdir()
1612+
oldBase64 = nothing
1613+
try
1614+
push!(empty!(DEPOT_PATH), depot_path)
1615+
Base64_key = Base.PkgId(Base.UUID("2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"), "Base64")
1616+
oldBase64 = Base.unreference_module(Base64_key)
1617+
cc = Base.compilecache(Base64_key)
1618+
@test Base.isprecompiled(Base64_key, cachepaths=String[cc[1]])
1619+
empty!(DEPOT_PATH)
1620+
Base.require_stdlib(Base64_key)
1621+
push!(DEPOT_PATH, depot_path)
1622+
append!(DEPOT_PATH, original_depot_path)
1623+
oldloaded = @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[])))
1624+
Base.require(Base64_key)
1625+
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded
1626+
Base.unreference_module(Base64_key)
1627+
empty!(DEPOT_PATH)
1628+
push!(DEPOT_PATH, depot_path)
1629+
Base.require(Base64_key)
1630+
@test @lock(Base.require_lock, length(get(Base.loaded_precompiles, Base64_key, Module[]))) == oldloaded + 1
1631+
Base.unreference_module(Base64_key)
1632+
finally
1633+
oldBase64 === nothing || Base.register_root_module(oldBase64)
1634+
copy!(DEPOT_PATH, original_depot_path)
1635+
rm(depot_path, force=true, recursive=true)
1636+
end
1637+
end

0 commit comments

Comments
 (0)