Skip to content

Commit 09d0eaa

Browse files
nh2nh2
committed
survey: Add package set that builds all stackage exes statically.
Also rework Haskell package sets to rebuild as few libraries as possible vs cache.nixos.org (see comments).
1 parent ef28327 commit 09d0eaa

File tree

1 file changed

+184
-47
lines changed

1 file changed

+184
-47
lines changed

‎survey/default.nix

Lines changed: 184 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,14 @@ let
2020
});
2121
};
2222

23+
normalPkgs = import <nixpkgs> {};
24+
2325
pkgs = (import <nixpkgs> {
26+
config.allowUnfree = true;
27+
config.allowBroken = true;
28+
# config.permittedInsecurePackages = [
29+
# "webkitgtk-2.4.11"
30+
# ];
2431
overlays = [ cabal2nix-fix-overlay ];
2532
}).pkgsMusl;
2633

@@ -33,76 +40,194 @@ let
3340

3441
normalHaskellPackages = pkgs.haskellPackages;
3542

36-
sqlite_static = pkgs.sqlite.overrideAttrs (old: { dontDisableStatic = true; });
3743

38-
lzma_static = pkgs.lzma.overrideAttrs (old: { dontDisableStatic = true; });
3944

40-
haskellPackages = with pkgs.haskell.lib; normalHaskellPackages.override (old: {
41-
overrides = pkgs.lib.composeExtensions (old.overrides or (_: _: {})) (self: super:
45+
lib = pkgs.lib;
46+
47+
# Function that tells us if a given Haskell package has an executable.
48+
isExecutable = pkg:
49+
(pkgs.haskell.lib.overrideCabal pkg (drv: {
50+
passthru.isExecutable = drv.isExecutable or false;
51+
})).isExecutable;
52+
53+
# Function that tells us if a given Haskell package is marked as broken.
54+
isBroken = pkg:
55+
(pkgs.haskell.lib.overrideCabal pkg (drv: {
56+
passthru.broken = drv.broken or false;
57+
})).broken;
58+
59+
# Function that for a given Haskell package tells us if any of
60+
# its dependencies is marked as `broken`.
61+
hasBrokenDeps = pkg:
4262
let
43-
# TODO do this via patches instead
44-
cabal_patched_src = pkgs.fetchFromGitHub {
45-
owner = "nh2";
46-
repo = "cabal";
47-
rev = "748f07b50724f2618798d200894f387020afc300";
48-
sha256 = "1k559m291f6spip50rly5z9rbxhfgzxvaz64cx4jqpxgfhbh2gfs";
49-
};
50-
51-
Cabal_patched_Cabal_subdir = pkgs.stdenv.mkDerivation {
52-
name = "cabal-dedupe-src";
53-
buildCommand = ''
54-
cp -rv ${cabal_patched_src}/Cabal/ $out
55-
'';
56-
};
57-
58-
Cabal_patched = self.callCabal2nix "Cabal" Cabal_patched_Cabal_subdir {};
59-
60-
useFixedCabal = drv: overrideCabal drv (old: {
61-
setupHaskellDepends = (if old ? setupHaskellDepends then old.setupHaskellDepends else []) ++ [ Cabal_patched ];
62-
# TODO Check if this is necessary
63-
libraryHaskellDepends = (if old ? libraryHaskellDepends then old.libraryHaskellDepends else []) ++ [ Cabal_patched ];
64-
});
63+
libraryDepends =
64+
(pkgs.haskell.lib.overrideCabal pkg (drv: {
65+
passthru.libraryHaskellDepends = drv.libraryHaskellDepends or [];
66+
})).libraryHaskellDepends;
67+
in
68+
lib.any (x:
69+
let
70+
res = builtins.tryEval (lib.isDerivation x && x ? env && isBroken x);
71+
broken = res.success && res.value;
72+
in
73+
if broken
74+
then builtins.trace "broken because of broken deps: ${pkg}" broken
75+
else broken
76+
) libraryDepends;
77+
78+
# Nixpkgs contains both Hackage and Stackage packages.
79+
# We want to build only executables that are on Stackage because
80+
# we know that those should build.
81+
# Find all Stackage package names here so we can use them
82+
# as a filter.
83+
# Done by parsing the configuration file that contains
84+
# which packages come from Stackage.
85+
stackagePackages =
86+
let
87+
stackageInfoPath = <nixpkgs/pkgs/development/haskell-modules/configuration-hackage2nix.yaml>;
88+
pythonWithYaml = pkgs.python2Packages.python.withPackages (pkgs: [pkgs.pyyaml]);
89+
dont-distribute-packages-file = normalPkgs.runCommand "test" {} ''
90+
${pythonWithYaml}/bin/python -c 'import yaml, json; x = yaml.load(open("${stackageInfoPath}")); print(json.dumps([line.split(" ")[0] for line in x["default-package-overrides"]]))' > $out
91+
'';
92+
dont-distribute-packages = builtins.fromJSON (builtins.readFile dont-distribute-packages-file);
93+
in
94+
dont-distribute-packages;
6595

66-
statify = drv: with pkgs.haskell.lib; pkgs.lib.foldl appendConfigureFlag (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) [
67-
# "--ghc-option=-fPIC"
68-
"--enable-executable-static" # requires `useFixedCabal`
69-
"--extra-lib-dirs=${pkgs.gmp6.override { withStatic = true; }}/lib"
70-
# TODO These probably shouldn't be here but only for packages that actually need them
71-
"--extra-lib-dirs=${pkgs.zlib.static}/lib"
72-
"--extra-lib-dirs=${pkgs.ncurses.override { enableStatic = true; }}/lib"
73-
];
96+
# Turns a list into a "set" (map where all keys are {}).
97+
keySet = list: builtins.listToAttrs (map (name: lib.nameValuePair name {}) list);
7498

99+
# Making it a set for faster lookup
100+
stackagePackagesSet = keySet stackagePackages;
101+
isStackagePackage = name: builtins.hasAttr name stackagePackagesSet;
102+
103+
# Stackage package names we want to blacklist.
104+
blacklist = [
105+
];
106+
107+
# All Stackage executables who (and whose dependencies) are not marked
108+
# as broken in nixpkgs.
109+
stackageExecutables = lib.filterAttrs (name: x: isStackagePackage name && !(lib.elem name blacklist) && (
110+
let
111+
res = builtins.tryEval (
112+
lib.isDerivation x
113+
&& x ? env
114+
&& isExecutable x
115+
&& !(isBroken x)
116+
&& !(hasBrokenDeps x)
117+
);
75118
in
76-
{
119+
res.success && res.value)
120+
) normalHaskellPackages;
121+
122+
# Making it a set for faster lookup
123+
stackageExecutablesSet = keySet (builtins.attrNames stackageExecutables);
124+
isStackageExecutable = name: builtins.hasAttr name stackageExecutablesSet;
125+
126+
numStackageExecutables = lib.length (builtins.attrNames stackageExecutables);
127+
128+
# Just for debugging / statistics:
129+
# Same thing with "-traced" suffix to nicely print
130+
# which executables we're going to build.
131+
stackageExecutablesNames = builtins.attrNames stackageExecutables;
132+
stackageExecutables-traced =
133+
builtins.trace
134+
("selected stackage executables:\n"
135+
+ lib.concatStringsSep "\n" stackageExecutablesNames
136+
+ "\n---\n${toString (lib.length stackageExecutablesNames)} executables total"
137+
)
138+
stackageExecutables;
139+
140+
141+
# Cherry-picking cabal fixes
142+
143+
# TODO Remove this when these fixes are available in nixpkgs:
144+
# https://github.com/haskell/cabal/pull/5356 (-L flag deduplication)
145+
# https://github.com/haskell/cabal/pull/5446 (--enable-executable-static)
146+
147+
# TODO do this via patches instead
148+
cabal_patched_src = pkgs.fetchFromGitHub {
149+
owner = "nh2";
150+
repo = "cabal";
151+
rev = "748f07b50724f2618798d200894f387020afc300";
152+
sha256 = "1k559m291f6spip50rly5z9rbxhfgzxvaz64cx4jqpxgfhbh2gfs";
153+
};
154+
155+
Cabal_patched_Cabal_subdir = pkgs.stdenv.mkDerivation {
156+
name = "cabal-dedupe-src";
157+
buildCommand = ''
158+
cp -rv ${cabal_patched_src}/Cabal/ $out
159+
'';
160+
};
161+
162+
Cabal_patched = normalHaskellPackages.callCabal2nix "Cabal" Cabal_patched_Cabal_subdir {};
163+
164+
useFixedCabal = drv: pkgs.haskell.lib.overrideCabal drv (old: {
165+
setupHaskellDepends = (if old ? setupHaskellDepends then old.setupHaskellDepends else []) ++ [ Cabal_patched ];
166+
# TODO Check if this is necessary
167+
libraryHaskellDepends = (if old ? libraryHaskellDepends then old.libraryHaskellDepends else []) ++ [ Cabal_patched ];
168+
});
169+
170+
171+
# Overriding system libraries that don't provide static libs
172+
# (`.a` files) by default
173+
174+
sqlite_static = pkgs.sqlite.overrideAttrs (old: { dontDisableStatic = true; });
175+
176+
lzma_static = pkgs.lzma.overrideAttrs (old: { dontDisableStatic = true; });
177+
178+
179+
# Overriding `haskellPackages` to fix *libraries* so that
180+
# they can be used in statically linked binaries.
181+
haskellPackagesWithLibsReadyForStaticLinking = with pkgs.haskell.lib; normalHaskellPackages.override (old: {
182+
overrides = pkgs.lib.composeExtensions (old.overrides or (_: _: {})) (self: super: {
183+
77184
# Helpers for other packages
78185

79186
hpc-coveralls = appendPatch super.hpc-coveralls (builtins.fetchurl https://github.com/guillaume-nargeot/hpc-coveralls/pull/73/commits/344217f513b7adfb9037f73026f5d928be98d07f.patch);
80187
persistent-sqlite = super.persistent-sqlite.override { sqlite = sqlite_static; };
81188
lzma = super.lzma.override { lzma = lzma_static; };
189+
82190
# If we `useFixedCabal` on stack, we also need to use the
83191
# it on hpack and hackage-security because otherwise
84192
# stack depends on 2 different versions of Cabal.
85193
hpack = useFixedCabal super.hpack;
86194
hackage-security = useFixedCabal super.hackage-security;
87195

88-
# Static executables that work
196+
# See https://github.com/hslua/hslua/issues/67
197+
# It's not clear if it's safe to disable this as key functionality may be broken
198+
hslua = dontCheck super.hslua;
89199

90-
hello = statify super.hello;
91-
hlint = statify super.hlint;
92-
ShellCheck = statify super.ShellCheck;
93-
cabal-install = statify super.cabal-install;
94-
bench = statify super.bench;
200+
});
95201

96-
stack = useFixedCabal (statify super.stack);
202+
});
97203

98-
dhall = statify super.dhall;
204+
statify = drv: with pkgs.haskell.lib; pkgs.lib.foldl appendConfigureFlag (disableLibraryProfiling (disableSharedExecutables (useFixedCabal drv))) [
205+
# "--ghc-option=-fPIC"
206+
"--enable-executable-static" # requires `useFixedCabal`
207+
"--extra-lib-dirs=${pkgs.gmp6.override { withStatic = true; }}/lib"
208+
# TODO These probably shouldn't be here but only for packages that actually need them
209+
"--extra-lib-dirs=${pkgs.zlib.static}/lib"
210+
"--extra-lib-dirs=${pkgs.ncurses.override { enableStatic = true; }}/lib"
211+
];
99212

100-
cachix = appendConfigureFlag (statify super.cachix) [ "--ghc-option=-j1" ];
213+
# Package set where all "final" executables are statically linked.
214+
#
215+
# In this package set, if executable E depends on package LE
216+
# which provides both a library and executables, then
217+
# E is statically linked but the executables of LE are not.
218+
#
219+
# Of course we could also make a different package set instead,
220+
# where executables from E and LE are all statically linked.
221+
# Then we would not need to distinguish between
222+
# `haskellPackages` and `haskellPackagesWithLibsReadyForStaticLinking`.
223+
# But we don't do that in order to cause as little needed rebuilding
224+
# of libraries vs cache.nixos.org as possible.
225+
haskellPackages =
226+
lib.mapAttrs (name: value:
227+
if isExecutable value then statify value else value
228+
) haskellPackagesWithLibsReadyForStaticLinking;
101229

102-
# Static executables that don't work yet
103230

104-
});
105-
});
106231

107232
in
108233
rec {
@@ -121,11 +246,23 @@ in
121246

122247
notWorking = {
123248
inherit (haskellPackages)
249+
xmonad
250+
pandoc
124251
;
125252
};
126253

127254
all = working // notWorking;
128255

256+
257+
# Tries to build all executables on Stackage.
258+
allStackageExecutables =
259+
lib.filterAttrs (name: x: isStackageExecutable name) haskellPackages;
260+
261+
262+
inherit normalPkgs;
263+
264+
inherit normalHaskellPackages;
265+
inherit haskellPackagesWithLibsReadyForStaticLinking;
129266
inherit haskellPackages;
130267
}
131268

0 commit comments

Comments
 (0)