Skip to content

Commit 0f81808

Browse files
andreabediniandreabedini
committed
Add separate cache for getPkgConfigDb
Querying pkg-config for the version of every module can be a very expensive operation on some systems. This change adds a separate, per-project, cache for PkgConfigDB; reducing the cost from "every plan change" to "every pkg-config-db change per project". The cache key is composed by the pkg-config configured program and the list of directories reported by pkg-config's pc_path variable. A notice is presented to the user when refreshing the PkgConfigDB.
1 parent b34184e commit 0f81808

File tree

152 files changed

+278
-58
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

152 files changed

+278
-58
lines changed

‎cabal-install/src/Distribution/Client/ProjectConfig.hs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ module Distribution.Client.ProjectConfig
5050
, resolveSolverSettings
5151
, BuildTimeSettings (..)
5252
, resolveBuildTimeSettings
53+
, resolveProgramDb
5354

5455
-- * Checking configuration
5556
, checkBadPerPackageCompilerPaths
@@ -154,6 +155,14 @@ import Distribution.Simple.InstallDirs
154155
)
155156
import Distribution.Simple.Program
156157
( ConfiguredProgram (..)
158+
, ProgramSearchPathEntry (..)
159+
)
160+
import Distribution.Simple.Program.Db
161+
( ProgramDb
162+
, defaultProgramDb
163+
, modifyProgramSearchPath
164+
, userSpecifyArgss
165+
, userSpecifyPaths
157166
)
158167
import Distribution.Simple.Setup
159168
( Flag (Flag)
@@ -496,6 +505,19 @@ resolveBuildTimeSettings
496505
| isParallelBuild buildSettingNumJobs = False
497506
| otherwise = False
498507

508+
-- | ProgramDb with directly user specified paths
509+
resolveProgramDb :: ProjectConfig -> ProgramDb
510+
resolveProgramDb ProjectConfig{projectConfigLocalPackages = PackageConfig{..}} =
511+
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
512+
. userSpecifyArgss (Map.toList (getMapMappend packageConfigProgramArgs))
513+
. modifyProgramSearchPath
514+
( ++
515+
[ ProgramSearchPathDir dir
516+
| dir <- fromNubList packageConfigProgramPathExtra
517+
]
518+
)
519+
$ defaultProgramDb
520+
499521
---------------------------------------------
500522
-- Reading and writing project config files
501523
--

‎cabal-install/src/Distribution/Client/ProjectPlanning.hs‎

Lines changed: 31 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@ configureCompiler
483483
DistDirLayout
484484
{ distProjectCacheFile
485485
}
486-
ProjectConfig
486+
projectConfig@ProjectConfig
487487
{ projectConfigShared =
488488
ProjectConfigShared
489489
{ projectConfigHcFlavor
@@ -533,15 +533,7 @@ configureCompiler
533533
hcFlavor = flagToMaybe projectConfigHcFlavor
534534
hcPath = flagToMaybe projectConfigHcPath
535535
hcPkg = flagToMaybe projectConfigHcPkg
536-
progdb =
537-
userSpecifyPaths (Map.toList (getMapLast packageConfigProgramPaths))
538-
. modifyProgramSearchPath
539-
( [ ProgramSearchPathDir dir
540-
| dir <- fromNubList packageConfigProgramPathExtra
541-
]
542-
++
543-
)
544-
$ defaultProgramDb
536+
progdb = resolveProgramDb projectConfig
545537

546538
------------------------------------------------------------------------------
547539

@@ -590,6 +582,9 @@ rebuildInstallPlan
590582
progsearchpath <- liftIO $ getSystemSearchPath
591583
let projectConfigMonitored = projectConfig{projectConfigBuildOnly = mempty}
592584

585+
let progdb = resolveProgramDb projectConfig
586+
pkgConfigDB <- getPkgConfigDb verbosity distDirLayout progdb
587+
593588
-- The overall improved plan is cached
594589
rerunIfChanged
595590
verbosity
@@ -609,11 +604,12 @@ rebuildInstallPlan
609604
)
610605
$ do
611606
compilerEtc <- phaseConfigureCompiler projectConfig
612-
_ <- phaseConfigurePrograms projectConfig compilerEtc
613-
(solverPlan, pkgConfigDB, totalIndexState, activeRepos) <-
607+
608+
(solverPlan, totalIndexState, activeRepos) <-
614609
phaseRunSolver
615610
projectConfig
616611
compilerEtc
612+
pkgConfigDB
617613
localPackages
618614
(fromMaybe mempty mbInstalledPackages)
619615
( elaboratedPlan
@@ -652,55 +648,37 @@ rebuildInstallPlan
652648
phaseConfigureCompiler
653649
:: ProjectConfig
654650
-> Rebuild (Compiler, Platform, ProgramDb)
655-
phaseConfigureCompiler = configureCompiler verbosity distDirLayout
651+
phaseConfigureCompiler projectConfig = do
652+
(compiler, platform, progdb) <- configureCompiler verbosity distDirLayout projectConfig
656653

657-
-- Configuring other programs.
658-
--
659-
-- Having configred the compiler, now we configure all the remaining
660-
-- programs. This is to check we can find them, and to monitor them for
661-
-- changes.
662-
--
663-
-- TODO: [required eventually] we don't actually do this yet.
664-
--
665-
-- We rely on the fact that the previous phase added the program config for
666-
-- all local packages, but that all the programs configured so far are the
667-
-- compiler program or related util programs.
668-
--
669-
phaseConfigurePrograms
670-
:: ProjectConfig
671-
-> (Compiler, Platform, ProgramDb)
672-
-> Rebuild ()
673-
phaseConfigurePrograms projectConfig (_, _, compilerprogdb) = do
674654
-- Users are allowed to specify program locations independently for
675655
-- each package (e.g. to use a particular version of a pre-processor
676656
-- for some packages). However they cannot do this for the compiler
677657
-- itself as that's just not going to work. So we check for this.
678658
liftIO $
679659
checkBadPerPackageCompilerPaths
680-
(configuredPrograms compilerprogdb)
660+
(configuredPrograms progdb)
681661
(getMapMappend (projectConfigSpecificPackage projectConfig))
682662

683-
-- TODO: [required eventually] find/configure other programs that the
684-
-- user specifies.
685-
686-
-- TODO: [required eventually] find/configure all build-tools
687-
-- but note that some of them may be built as part of the plan.
663+
return (compiler, platform, progdb)
688664

689665
-- Run the solver to get the initial install plan.
690666
-- This is expensive so we cache it independently.
691667
--
692668
phaseRunSolver
693669
:: ProjectConfig
694670
-> (Compiler, Platform, ProgramDb)
671+
-> PkgConfigDb
695672
-> [PackageSpecifier UnresolvedSourcePackage]
696673
-> InstalledPackageIndex
697-
-> Rebuild (SolverInstallPlan, PkgConfigDb, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
674+
-> Rebuild (SolverInstallPlan, IndexUtils.TotalIndexState, IndexUtils.ActiveRepos)
698675
phaseRunSolver
699676
projectConfig@ProjectConfig
700677
{ projectConfigShared
701678
, projectConfigBuildOnly
702679
}
703680
(compiler, platform, progdb)
681+
pkgConfigDB
704682
localPackages
705683
installedPackages =
706684
rerunIfChanged
@@ -721,13 +699,13 @@ rebuildInstallPlan
721699
progdb
722700
platform
723701
corePackageDbs
702+
724703
(sourcePkgDb, tis, ar) <-
725704
getSourcePackages
726705
verbosity
727706
withRepoCtx
728707
(solverSettingIndexState solverSettings)
729708
(solverSettingActiveRepos solverSettings)
730-
pkgConfigDB <- getPkgConfigDb verbosity progdb
731709

732710
-- TODO: [code cleanup] it'd be better if the Compiler contained the
733711
-- ConfiguredPrograms that it needs, rather than relying on the progdb
@@ -752,7 +730,7 @@ rebuildInstallPlan
752730
Left msg -> do
753731
reportPlanningFailure projectConfig compiler platform localPackages
754732
dieWithException verbosity $ PhaseRunSolverErr msg
755-
Right plan -> return (plan, pkgConfigDB, tis, ar)
733+
Right plan -> return (plan, tis, ar)
756734
where
757735
corePackageDbs :: [PackageDB]
758736
corePackageDbs =
@@ -1038,13 +1016,21 @@ getSourcePackages verbosity withRepoCtx idxState activeRepos = do
10381016
$ repos
10391017
return sourcePkgDbWithTIS
10401018

1041-
getPkgConfigDb :: Verbosity -> ProgramDb -> Rebuild PkgConfigDb
1042-
getPkgConfigDb verbosity progdb = do
1019+
getPkgConfigDb :: Verbosity -> DistDirLayout -> ProgramDb -> Rebuild PkgConfigDb
1020+
getPkgConfigDb verbosity distDirLayout progdb = do
1021+
-- We find the pkg-config program and we ask it for the location of the pc files.
1022+
-- We then use these two values for the cache key
1023+
mpkgConfig <- liftIO $ needProgram verbosity pkgConfigProgram progdb
1024+
let mpkgConfigLoc = fst <$> mpkgConfig
10431025
dirs <- liftIO $ getPkgConfigDbDirs verbosity progdb
1044-
-- Just monitor the dirs so we'll notice new .pc files.
1045-
-- Alternatively we could monitor all the .pc files too.
1046-
traverse_ monitorDirectoryStatus dirs
1047-
liftIO $ readPkgConfigDb verbosity progdb
1026+
rerunIfChanged verbosity fileMonitorPkgConfigDb (mpkgConfigLoc, dirs) $ do
1027+
liftIO $ notice verbosity "Querying pkg-config database..."
1028+
-- Just monitor the dirs so we'll notice new .pc files.
1029+
-- Alternatively we could monitor all the .pc files too.
1030+
traverse_ monitorDirectoryStatus dirs
1031+
liftIO $ readPkgConfigDb verbosity progdb
1032+
where
1033+
fileMonitorPkgConfigDb = newFileMonitor $ distProjectCacheFile distDirLayout "pkg-config-db"
10481034

10491035
-- | Select the config values to monitor for changes package source hashes.
10501036
packageLocationsSignature

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external-target.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-external.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal-target.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Build profile: -w ghc-<GHCVER> -O1
45
In order, the following will be built:
@@ -18,8 +19,7 @@ Configuring library 'indef' instantiated with
1819
Data.Map = containers-<VERSION>:Data.Map
1920
for Includes3-0.1.0.0...
2021
Preprocessing library 'indef' for Includes3-0.1.0.0...
21-
Building library 'indef' instantiated with
22-
Data.Map = containers-<VERSION>:Data.Map
22+
Building library 'indef' instantiated with Data.Map = containers-<VERSION>:Data.Map
2323
for Includes3-0.1.0.0...
2424
Configuring executable 'exe' for Includes3-0.1.0.0...
2525
Preprocessing executable 'exe' for Includes3-0.1.0.0...

‎cabal-testsuite/PackageTests/Backpack/Includes3/cabal-repo.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# cabal v2-update
22
Downloading the latest package list from test-local-repo
33
# cabal v2-build
4+
Querying pkg-config database...
45
Resolving dependencies...
56
Build profile: -w ghc-<GHCVER> -O1
67
In order, the following will be built:

‎cabal-testsuite/PackageTests/Backpack/Reexport2/cabal.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# cabal v2-build
2+
Querying pkg-config database...
23
Resolving dependencies...
34
Error:
45
Problem with module re-exports:

0 commit comments

Comments
 (0)