Skip to content

Commit a4fce1f

Browse files
Anton-LatukhaAnton-Latukha
committed
CI: {caching,test,bench,hackage}: new caching process
Sources are stored separately. Before this cache was storing per Platform/GHC sources & compiled results together. While deps sources does not depend on platform or GHC, they are platform & GHC agnostic. Because sources & compiled results were stored togather & sources took ~1/2-2/3 of that - cache stored multiple duplicates of the same source code. This config stores the source code separately. And at the same time tries to store & share all source code which may be needed. & all compiled deps that would be used in different workflows.
1 parent ee64445 commit a4fce1f

File tree

4 files changed

+75
-40
lines changed

4 files changed

+75
-40
lines changed

‎.github/workflows/bench.yml

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,12 @@ jobs:
4747
- run: git fetch origin master # check the master branch for benchmarking
4848

4949
- uses: haskell/actions/setup@v1
50+
id: HaskEnvSetup
5051
with:
5152
ghc-version : ${{ matrix.ghc }}
5253
cabal-version: ${{ matrix.cabal }}
5354
enable-stack: false
5455

55-
- name: Cache Cabal
5656
- name: Linux Platform config
5757
run: |
5858
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
@@ -65,16 +65,26 @@ jobs:
6565
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
6666
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
6767
68+
- name: Hackage sources cache
6869
uses: actions/cache@v2
70+
env:
71+
cache-name: hackage-sources
6972
with:
70-
path: |
71-
~/.cabal/packages
72-
~/.cabal/store
73-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
73+
path: ${{ env.CABAL_PKGS_DIR }}
74+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
75+
restore-keys: ${{ env.cache-name }}-
76+
77+
- name: Compiled deps cache
78+
uses: actions/cache@v2
79+
env:
80+
cache-name: compiled-deps
81+
with:
82+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
83+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
7484
restore-keys: |
75-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
76-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-
77-
v2-${{ runner.os }}-${{ matrix.ghc }}
85+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
86+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
87+
${{ env.cache-name }}-${{ runner.os }}-
7888
7989
- run: cabal update
8090

‎.github/workflows/caching.yml

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -66,20 +66,19 @@ jobs:
6666
- uses: actions/checkout@v2
6767

6868
- uses: haskell/actions/setup@v1
69+
id: HaskEnvSetup
6970
with:
70-
ghc-version: ${{ matrix.ghc }}
71+
ghc-version : ${{ matrix.ghc }}
7172
cabal-version: ${{ matrix.cabal }}
7273
enable-stack: false
7374

7475
- if: runner.os == 'Windows'
7576
name: (Windows) Platform config
7677
run: |
77-
echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV
7878
echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV
7979
- if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' )
8080
name: (Linux,macOS) Platform config
8181
run: |
82-
echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV
8382
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
8483
8584
# Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file
@@ -93,7 +92,6 @@ jobs:
9392
echo "package floskell" >> cabal.project
9493
echo " ghc-options: -O0" >> cabal.project
9594
96-
- name: Cache Cabal
9795
- name: Retrieving `cabal.project` Hackage timestamp
9896
run: |
9997
# Form: index-state: 2021-11-29T08:11:08Z
@@ -102,18 +100,31 @@ jobs:
102100
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
103101
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
104102
103+
# 2021-12-02: NOTE: Cabal Hackage source tree storage does not depend on OS or GHC really,
104+
# but can depend on `base`.
105+
# But this caching is happens only inside `master` for `master` purposes of compiling the deps
106+
# so having a shared pool here that depends only on Hackage pin & does not depend on `base` is "good enough"
107+
# & used such because it preserves 10% of a global cache storage pool.
108+
- name: Hackage sources cache
105109
uses: actions/cache@v2
106110
env:
107-
cache-name: cache-cabal
111+
cache-name: hackage-sources
108112
with:
109-
path: |
110-
${{ env.CABAL_PKGS_DIR }}
111-
${{ env.CABAL_STORE_DIR }}
112-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
113+
path: ${{ env.CABAL_PKGS_DIR }}
114+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
115+
restore-keys: ${{ env.cache-name }}-
116+
117+
- name: Compiled deps cache
118+
uses: actions/cache@v2
119+
env:
120+
cache-name: compiled-deps
121+
with:
122+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
123+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
113124
restore-keys: |
114-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
115-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
116-
v2-${{ runner.os }}-${{ matrix.ghc }}
125+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
126+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
127+
${{ env.cache-name }}-${{ runner.os }}-
117128
118129
- run: cabal update
119130

‎.github/workflows/hackage.yml

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,12 @@ jobs:
4949
- uses: actions/checkout@v2
5050

5151
- uses: haskell/actions/setup@v1
52+
id: HaskEnvSetup
5253
with:
5354
ghc-version : ${{ matrix.ghc }}
5455
cabal-version: ${{ matrix.cabal }}
5556
enable-stack: false
5657

57-
- name: Cache Cabal
5858
- name: Linux Platform config
5959
run: |
6060
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
@@ -67,18 +67,26 @@ jobs:
6767
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
6868
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
6969
70+
- name: Hackage sources cache
7071
uses: actions/cache@v2
7172
env:
72-
cache-name: cache-cabal
73+
cache-name: hackage-sources
7374
with:
74-
path: |
75-
~/.cabal/packages
76-
~/.cabal/store
77-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
75+
path: ${{ env.CABAL_PKGS_DIR }}
76+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
77+
restore-keys: ${{ env.cache-name }}-
78+
79+
- name: Compiled deps cache
80+
uses: actions/cache@v2
81+
env:
82+
cache-name: compiled-deps
83+
with:
84+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
85+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
7886
restore-keys: |
79-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
80-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
81-
v2-${{ runner.os }}-${{ matrix.ghc }}
87+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
88+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
89+
${{ env.cache-name }}-${{ runner.os }}-
8290
8391
- name: "Run cabal check"
8492
run: |

‎.github/workflows/test.yml

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ jobs:
7676
- uses: actions/checkout@v2
7777

7878
- uses: haskell/actions/setup@v1
79+
id: HaskEnvSetup
7980
with:
8081
ghc-version : ${{ matrix.ghc }}
8182
cabal-version: ${{ matrix.cabal }}
@@ -84,12 +85,10 @@ jobs:
8485
- if: runner.os == 'Windows'
8586
name: (Windows) Platform config
8687
run: |
87-
echo "CABAL_STORE_DIR=$SYSTEMDRIVE\\SR" >> $GITHUB_ENV
8888
echo "CABAL_PKGS_DIR=~\\AppData\\cabal\\packages" >> $GITHUB_ENV
8989
- if: ( runner.os == 'Linux' ) || ( runner.os == 'macOS' )
9090
name: (Linux,macOS) Platform config
9191
run: |
92-
echo "CABAL_STORE_DIR=~/.cabal/store" >> $GITHUB_ENV
9392
echo "CABAL_PKGS_DIR=~/.cabal/packages" >> $GITHUB_ENV
9493
9594
# Needs to be before Cache Cabal so the cache can detect changes to the modified cabal.project file
@@ -114,7 +113,6 @@ jobs:
114113
sed -i.bak -e 's/Paths_haskell_language_server/Paths_hls/g' \
115114
src/**/*.hs exe/*.hs
116115
117-
- name: Cache Cabal
118116
- name: Retrieving `cabal.project` Hackage timestamp
119117
run: |
120118
# Form: index-state: 2021-11-29T08:11:08Z
@@ -123,18 +121,26 @@ jobs:
123121
INDEX_STATE1=$(echo "$INDEX_STATE_ENTRY" | cut -d' ' -f2 | tr ':' '-')
124122
echo "INDEX_STATE=$INDEX_STATE1" >> $GITHUB_ENV
125123
124+
- name: Hackage sources cache
126125
uses: actions/cache@v2
127126
env:
128-
cache-name: cache-cabal
127+
cache-name: hackage-sources
129128
with:
130-
path: |
131-
${{ env.CABAL_PKGS_DIR }}
132-
${{ env.CABAL_STORE_DIR }}
133-
key: v2-${{ runner.os }}-${{ matrix.ghc }}-build-${{ hashFiles('cabal.project') }}
129+
path: ${{ env.CABAL_PKGS_DIR }}
130+
key: ${{ env.cache-name }}-${{ env.INDEX_STATE }}
131+
restore-keys: ${{ env.cache-name }}-
132+
133+
- name: Compiled deps cache
134+
uses: actions/cache@v2
135+
env:
136+
cache-name: compiled-deps
137+
with:
138+
path: ${{ steps.HaskEnvSetup.outputs.cabal-store }}
139+
key: ${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-${{ hashFiles('cabal.project') }}
134140
restore-keys: |
135-
v2-${{ runner.os }}-${{ matrix.ghc }}-bench-${{ hashFiles('cabal.project') }}
136-
v2-${{ runner.os }}-${{ matrix.ghc }}-build-
137-
v2-${{ runner.os }}-${{ matrix.ghc }}
141+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-${{ env.INDEX_STATE }}-
142+
${{ env.cache-name }}-${{ runner.os }}-${{ matrix.ghc }}-
143+
${{ env.cache-name }}-${{ runner.os }}-
138144
139145
- run: cabal update
140146

0 commit comments

Comments
 (0)