Add incremental builds on startup using cached dependency graph

lib/src/asset/cache.dart

@@ -49,6 +49,11 @@ class CachedAssetReader extends AssetReader {
 
   CachedAssetReader(this._cache, this._reader);
 
+  /// Evicts [id] from the underlying cache.
+  void evictFromCache(AssetId id) {
+    _cache.remove(id);
+  }
+
   @override
   Future<bool> hasInput(AssetId id) {
     if (_cache.contains(id)) return new Future.value(true);

lib/src/asset_graph/graph.dart

@@ -10,13 +10,21 @@ class AssetGraph {
   /// All the [AssetNode]s in the system, indexed by [AssetId].
   final _nodesById = <AssetId, AssetNode>{};
 
+  /// This represents start time of the most recent build which created this
+  /// graph. Any assets which have been updated after this time should be
+  /// invalidated on subsequent builds.
+  ///
+  /// This is initialized to a very old value, and should be set to a real
+  /// value if you want incremental rebuilds.
+  DateTime validAsOf = new DateTime.fromMillisecondsSinceEpoch(0);
+
   AssetGraph();
 
   /// Part of the serialized graph, used to ensure versioning constraints.
   ///
   /// This should be incremented any time the serialize/deserialize methods
   /// change on this class or [AssetNode].
-  static get _version => 1;
+  static int get _version => 2;
 
   /// Deserializes this graph.
   factory AssetGraph.deserialize(Map serializedGraph) {
@@ -29,13 +37,16 @@ class AssetGraph {
     for (var serializedItem in serializedGraph['nodes']) {
       graph.add(new AssetNode.deserialize(serializedItem));
     }
+    graph.validAsOf =
+        new DateTime.fromMillisecondsSinceEpoch(serializedGraph['validAsOf']);
     return graph;
   }
 
   /// Puts this graph into a serializable form.
   Map serialize() => {
         'version': _version,
         'nodes': allNodes.map((node) => node.serialize()).toList(),
+        'validAsOf': validAsOf.millisecondsSinceEpoch,
       };
 
   /// Checks if [id] exists in the graph.
@@ -66,5 +77,5 @@ class AssetGraph {
   Iterable<AssetNode> get allNodes => _nodesById.values;
 
   @override
-  toString() => _nodesById.values.toList().toString();
+  toString() => 'validAsOf: $validAsOf\n${_nodesById.values.toList()}';
 }

lib/src/generate/build.dart

@@ -102,8 +102,8 @@ Stream<BuildResult> watch(List<List<Phase>> phaseGroups,
   writer ??=
       new CachedAssetWriter(cache, new FileBasedAssetWriter(packageGraph));
   directoryWatcherFactory ??= defaultDirectoryWatcherFactory;
-  var watchImpl = new WatchImpl(directoryWatcherFactory, debounceDelay, cache,
-      reader, writer, packageGraph, phaseGroups);
+  var watchImpl = new WatchImpl(directoryWatcherFactory, debounceDelay, reader,
+      writer, packageGraph, phaseGroups);
 
   var resultStream = watchImpl.runWatch();
 

lib/src/generate/build_impl.dart

@@ -7,10 +7,12 @@ import 'dart:io';
 
 import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
+import 'package:watcher/watcher.dart';
 
 import '../asset/asset.dart';
 import '../asset/exceptions.dart';
 import '../asset/id.dart';
+import '../asset/cache.dart';
 import '../asset/reader.dart';
 import '../asset/writer.dart';
 import '../asset_graph/exceptions.dart';
@@ -46,7 +48,14 @@ class BuildImpl {
   /// [BuildStatus.Failure]. The exception and stack trace that caused the failure
   /// will be available as [BuildResult#exception] and [BuildResult#stackTrace]
   /// respectively.
-  Future<BuildResult> runBuild() async {
+  ///
+  /// The [validAsOf] date is assigned to [AssetGraph#validAsOf] and marks
+  /// a point in time after which any updates to files should invalidate the
+  /// graph for future builds.
+  Future<BuildResult> runBuild(
+      {DateTime validAsOf, Map<AssetId, ChangeType> updates}) async {
+    validAsOf ??= new DateTime.now();
+    updates ??= <AssetId, ChangeType>{};
     try {
       if (_buildRunning) throw const ConcurrentBuildException();
       _buildRunning = true;
@@ -55,8 +64,18 @@ class BuildImpl {
       if (_assetGraph == null) {
         _logger.info('Reading cached dependency graph');
         _assetGraph = await _readAssetGraph();
+
+        /// Collect updates since the asset graph was last created. This only
+        /// handles updates and deletes, not adds. We list the file system for
+        /// all inputs later on (in [_initializeInputsByPackage]).
+        updates.addAll(await _getUpdates());
       }
 
+      /// Applies all [updates] to the [_assetGraph] as well as doing other
+      /// necessary cleanup.
+      await _updateWithChanges(updates);
+      _assetGraph.validAsOf = validAsOf;
+
       /// Wait while all inputs are collected.
       _logger.info('Initializing inputs');
       await _initializeInputsByPackage();
@@ -91,14 +110,8 @@ class BuildImpl {
   Future<AssetGraph> _readAssetGraph() async {
     if (!await _reader.hasInput(_assetGraphId)) return new AssetGraph();
     try {
-      _logger.info('Reading cached asset graph.');
-      var graph = new AssetGraph.deserialize(
+      return new AssetGraph.deserialize(
           JSON.decode(await _reader.readAsString(_assetGraphId)));
-      /// TODO(jakemac): Only invalidate nodes which need invalidating, which
-      /// will give us incremental builds on startup.
-      graph.allNodes.where((node) => node is GeneratedAssetNode)
-          .forEach((node) => node.needsUpdate = true);
-      return graph;
     } on AssetGraphVersionException catch (_) {
       /// Start fresh if the cached asset_graph version doesn't match up with
       /// the current version. We don't currently support old graph versions.
@@ -107,13 +120,67 @@ class BuildImpl {
     }
   }
 
-  /// Deletes all previous output files.
+  /// Creates and returns a map of updates to assets based on [_assetGraph].
+  Future<Map<AssetId, ChangeType>> _getUpdates() async {
+    /// Collect updates to the graph based on any changed assets.
+    var updates = <AssetId, ChangeType>{};
+    await Future.wait(_assetGraph.allNodes
+        .where((node) => node is! GeneratedAssetNode)
+        .map((node) async {
+      if (await _reader.hasInput(node.id)) {
+        var lastModified = await _reader.lastModified(node.id);
+        if (lastModified.compareTo(_assetGraph.validAsOf) > 0) {
+          updates[node.id] = ChangeType.MODIFY;
+        }
+      } else {
+        updates[node.id] = ChangeType.REMOVE;
+      }
+    }));
+    return updates;
+  }
+
+  /// Applies all [updates] to the [_assetGraph] as well as doing other
+  /// necessary cleanup such as clearing caches for [CachedAssetReader]s and
+  /// deleting outputs as necessary.
+  Future _updateWithChanges(Map<AssetId, ChangeType> updates) async {
+    Future clearNodeAndDeps(AssetId id, ChangeType rootChangeType,
+        {AssetId parent}) async {
+      var node = _assetGraph.get(id);
+      if (node == null) return;
+      if (_reader is CachedAssetReader) {
+        (_reader as CachedAssetReader).evictFromCache(id);
+      }
+
+      /// Update all ouputs of this asset as well.
+      await Future.wait(node.outputs.map((output) =>
+          clearNodeAndDeps(output, rootChangeType, parent: node.id)));
+
+      /// For deletes, prune the graph.
+      if (parent == null && rootChangeType == ChangeType.REMOVE) {
+        _assetGraph.remove(id);
+      }
+      if (node is GeneratedAssetNode) {
+        node.needsUpdate = true;
+        if (rootChangeType == ChangeType.REMOVE &&
+            node.primaryInput == parent) {
+          _assetGraph.remove(id);
+          await _writer.delete(id);
+        }
+      }
+    }
+
+    await Future.wait(
+        updates.keys.map((input) => clearNodeAndDeps(input, updates[input])));
+  }
+
+  /// Deletes all previous output files that are in need of an update.
   Future _deletePreviousOutputs() async {
     /// TODO(jakemac): need a cleaner way of telling if the current graph was
     /// generated from cache or if its just a brand new graph.
     if (await _reader.hasInput(_assetGraphId)) {
       await _writer.delete(_assetGraphId);
       _inputsByPackage[_assetGraphId.package]?.remove(_assetGraphId);
+
       /// Remove all output nodes from [_inputsByPackage], and delete all assets
       /// that need updates.
       await Future.wait(_assetGraph.allNodes

lib/src/generate/watch_impl.dart

@@ -25,9 +25,6 @@ class WatchImpl {
   /// All file listeners currently set up.
   final _allListeners = <StreamSubscription>[];
 
-  /// The [AssetCache] being used for builds.
-  final AssetCache _assetCache;
-
   /// The [AssetGraph] being shared with [_buildImpl]
   AssetGraph get _assetGraph => _buildImpl.assetGraph;
 
@@ -67,7 +64,6 @@ class WatchImpl {
   WatchImpl(
       this._directoryWatcherFactory,
       this._debounceDelay,
-      this._assetCache,
       AssetReader reader,
       AssetWriter writer,
       PackageGraph packageGraph,
@@ -127,6 +123,10 @@ class WatchImpl {
       }
       assert(_nextBuildScheduled == false);
 
+      /// Any updates after this point should cause updates to the [AssetGraph]
+      /// for later builds.
+      var validAsOf = new DateTime.now();
+
       /// Copy [updatedInputs] so that it doesn't get modified after this point.
       /// Any further updates will be scheduled for the next build.
       ///
@@ -142,49 +142,21 @@ class WatchImpl {
       }
 
       _logger.info('Preparing for next build');
-      Future clearNodeAndDeps(
-          AssetId id, AssetId primaryInput, ChangeType rootChangeType) async {
-        var node = _assetGraph.get(id);
-        if (node == null) return;
-        _assetCache.remove(id);
-
-        /// Update all ouputs of this asset as well.
-        await Future.wait(node.outputs.map((output) =>
-            clearNodeAndDeps(output, primaryInput, rootChangeType)));
-
-        /// For deletes, prune the graph.
-        if (id == primaryInput && rootChangeType == ChangeType.REMOVE) {
-          _assetGraph.remove(id);
+      _logger.info('Starting build');
+      _currentBuild =
+          _buildImpl.runBuild(validAsOf: validAsOf, updates: updatedInputsCopy);
+      _currentBuild.then((result) {
+        if (result.status == BuildStatus.Success) {
+          _logger.info('Build completed successfully');
+        } else {
+          _logger.warning('Build failed');
         }
-        if (node is GeneratedAssetNode) {
-          node.needsUpdate = true;
-          if (rootChangeType == ChangeType.REMOVE &&
-              node.primaryInput == primaryInput) {
-            _assetGraph.remove(id);
-            await _writer.delete(id);
-          }
+        _resultStreamController.add(result);
+        _currentBuild = null;
+        if (_nextBuildScheduled) {
+          _nextBuildScheduled = false;
+          doBuild();
         }
-      }
-
-      Future
-          .wait(updatedInputsCopy.keys.map((input) =>
-              clearNodeAndDeps(input, input, updatedInputsCopy[input])))
-          .then((_) {
-        _logger.info('Starting build');
-        _currentBuild = _buildImpl.runBuild();
-        _currentBuild.then((result) {
-          if (result.status == BuildStatus.Success) {
-            _logger.info('Build completed successfully');
-          } else {
-            _logger.warning('Build failed');
-          }
-          _resultStreamController.add(result);
-          _currentBuild = null;
-          if (_nextBuildScheduled) {
-            _nextBuildScheduled = false;
-            doBuild();
-          }
-        });
       });
     }
 

test/asset_graph/graph_test.dart

@@ -15,7 +15,7 @@ main() {
     AssetGraph graph;
 
     setUp(() {
-      graph = new AssetGraph();
+      graph = new AssetGraph()..validAsOf = new DateTime.now();
     });
 
     void expectNodeDoesNotExist(AssetNode node) {

test/common/matchers.dart

@@ -35,17 +35,22 @@ class _AssetMatcher extends Matcher {
       description.addDescriptionOf(_expected);
 }
 
-equalsAssetGraph(AssetGraph expected) => new _AssetGraphMatcher(expected);
+equalsAssetGraph(AssetGraph expected, {bool checkValidAsOf}) =>
+    new _AssetGraphMatcher(expected, checkValidAsOf ?? false);
 
 class _AssetGraphMatcher extends Matcher {
   final AssetGraph _expected;
+  final bool _checkValidAsOf;
 
-  const _AssetGraphMatcher(this._expected);
+  const _AssetGraphMatcher(this._expected, this._checkValidAsOf);
 
   @override
   bool matches(item, _) {
     if (item is! AssetGraph) return false;
     if (item.allNodes.length != _expected.allNodes.length) return false;
+    if (_checkValidAsOf && (item.validAsOf != _expected.validAsOf)) {
+      return false;
+    }
     for (var node in item.allNodes) {
       var expectedNode = _expected.get(node.id);
       if (expectedNode == null || expectedNode.id != node.id) return false;