diff --git a/cabal.project b/cabal.project
index b0fbffe75d..03cca1845b 100644
--- a/cabal.project
+++ b/cabal.project
@@ -55,17 +55,6 @@ allow-newer:
     deepseq:base,
     dependent-sum:some,
     dependent-sum:constraints,
-    diagrams-contrib:base,
-    diagrams-contrib:lens,
-    diagrams-contrib:random,
-    diagrams-core:base,
-    diagrams-core:lens,
-    diagrams-lib:base,
-    diagrams-lib:lens,
-    diagrams-postscript:base,
-    diagrams-postscript:lens,
-    diagrams-svg:base,
-    diagrams-svg:lens,
     dual-tree:base,
     -- Does this make any sense?
     entropy:Cabal,
@@ -81,7 +70,9 @@ allow-newer:
     regex-base:base,
     regex-tdfa:base,
     statestack:base,
-    svg-builder:base,
     these:base,
-    time-compat:base
+    time-compat:base,
 
+    -- for shake-bench
+    Chart-diagrams:diagrams-core,
+    SVGFonts:diagrams-core
diff --git a/shake-bench/shake-bench.cabal b/shake-bench/shake-bench.cabal
index b966907cce..6dea31665f 100644
--- a/shake-bench/shake-bench.cabal
+++ b/shake-bench/shake-bench.cabal
@@ -20,7 +20,9 @@ library
         base == 4.*,
         Chart,
         Chart-diagrams,
-        diagrams,
+        diagrams-contrib,
+        diagrams-core,
+        diagrams-lib,
         diagrams-svg,
         directory,
         extra >= 1.7.2,