Skip to content

Commit 47ea86a

Browse files
authored
Content aware hash moved to script and tracked (flutter#166717)
1. Calculate the hash in only two places: `content_aware_hash.{ps1|sh}` 2. Call this from the workflow 3. Eventually call this from `update_engine_version.{ps1|sh}` The files of import: * `DEPS`: tracks third party dependencies related to building the engine * `engine`: all the code in the engine folder * `bin/internal/content_aware_hash.ps1`: script for calculating the hash on windows * `bin/internal/content_aware_hash.sh`: script for calculating the hash on mac/linux * `.github/workflows/content-aware-hash.yml`: github action for CI/CD hashing Tested on windows and mac: ```shell PS C:\src\flutter> C:\src\flutter\bin\internal\content_aware_hash.ps1 c24231e276e0719738e175e0622e040ad21a7012 ``` ```shell ❯ ~/src/flutter/bin/internal/content_aware_hash.sh c24231e276e0719738e175e0622e040ad21a7012 ```
1 parent 0321ef5 commit 47ea86a

File tree

6 files changed

+377
-1
lines changed

6 files changed

+377
-1
lines changed

.github/workflows/content-aware-hash.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
2020
- name: Generate Hash
2121
run: |
22-
engine_content_hash=$(git ls-tree HEAD DEPS bin/internal/release-candidate-branch.version engine | git hash-object --stdin)
22+
engine_content_hash=$(bin/internal/content_aware_hash.sh)
2323
# test notice annotation for retrival from api
2424
echo "::notice ::{\"engine_content_hash\": \"${engine_content_hash}\"}"
2525
# test summary writing

bin/internal/content_aware_hash.ps1

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Copyright 2014 The Flutter Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style license that can be
3+
# found in the LICENSE file.
4+
5+
# ---------------------------------- NOTE ---------------------------------- #
6+
#
7+
# Please keep the logic in this file consistent with the logic in the
8+
# `content_aware_hash.ps1` script in the same directory to ensure that Flutter
9+
# continues to work across all platforms!
10+
#
11+
# -------------------------------------------------------------------------- #
12+
13+
$ErrorActionPreference = "Stop"
14+
15+
# When called from a submodule hook; these will override `git -C dir`
16+
$env:GIT_DIR = $null
17+
$env:GIT_INDEX_FILE = $null
18+
$env:GIT_WORK_TREE = $null
19+
20+
$progName = Split-Path -parent $MyInvocation.MyCommand.Definition
21+
$flutterRoot = (Get-Item $progName).parent.parent.FullName
22+
23+
# Cannot use '*' for files in this command
24+
# DEPS: tracks third party dependencies related to building the engine
25+
# engine: all the code in the engine folder
26+
# bin/internal/content_aware_hash.ps1: script for calculating the hash on windows
27+
# bin/internal/content_aware_hash.sh: script for calculating the hash on mac/linux
28+
# .github/workflows/content-aware-hash.yml: github action for CI/CD hashing
29+
cmd /c "git -C ""$flutterRoot"" ls-tree --format ""%(objectname) %(path)"" HEAD DEPS engine bin/internal/release-candidate-branch.version | git hash-object --stdin"

bin/internal/content_aware_hash.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2014 The Flutter Authors. All rights reserved.
3+
# Use of this source code is governed by a BSD-style license that can be
4+
# found in the LICENSE file.
5+
6+
# ---------------------------------- NOTE ---------------------------------- #
7+
#
8+
# Please keep the logic in this file consistent with the logic in the
9+
# `content_aware_hash.ps1` script in the same directory to ensure that Flutter
10+
# continues to work across all platforms!
11+
#
12+
# -------------------------------------------------------------------------- #
13+
14+
set -e
15+
16+
FLUTTER_ROOT="$(dirname "$(dirname "$(dirname "${BASH_SOURCE[0]}")")")"
17+
18+
unset GIT_DIR
19+
unset GIT_INDEX_FILE
20+
unset GIT_WORK_TREE
21+
22+
# Cannot use '*' for files in this command
23+
# DEPS: tracks third party dependencies related to building the engine
24+
# engine: all the code in the engine folder
25+
# bin/internal/content_aware_hash.ps1: script for calculating the hash on windows
26+
# bin/internal/content_aware_hash.sh: script for calculating the hash on mac/linux
27+
# .github/workflows/content-aware-hash.yml: github action for CI/CD hashing
28+
git -C "$FLUTTER_ROOT" ls-tree --format "%(objectname) %(path)" HEAD DEPS engine bin/internal/release-candidate-branch.version | git hash-object --stdin

dev/bots/analyze.dart

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2599,6 +2599,7 @@ const Set<String> kExecutableAllowlist = <String>{
25992599
'bin/flutter-dev',
26002600
'bin/internal/update_dart_sdk.sh',
26012601
'bin/internal/update_engine_version.sh',
2602+
'bin/internal/content_aware_hash.sh',
26022603

26032604
'dev/bots/codelabs_build_test.sh',
26042605
'dev/bots/docs.sh',
Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
@TestOn('vm')
6+
library;
7+
8+
import 'dart:io' as io;
9+
10+
import 'package:file/file.dart';
11+
import 'package:file/local.dart';
12+
import 'package:platform/platform.dart';
13+
import 'package:test/test.dart';
14+
15+
//////////////////////////////////////////////////////////////////////
16+
// //
17+
// ✨ THINKING OF MOVING/REFACTORING THIS FILE? READ ME FIRST! ✨ //
18+
// //
19+
// There is a link to this file in //docs/tool/Engine-artfiacts.md //
20+
// and it would be very kind of you to update the link, if needed. //
21+
// //
22+
//////////////////////////////////////////////////////////////////////
23+
24+
void main() {
25+
const FileSystem localFs = LocalFileSystem();
26+
final _FlutterRootUnderTest flutterRoot = _FlutterRootUnderTest.findWithin();
27+
28+
late Directory tmpDir;
29+
late _FlutterRootUnderTest testRoot;
30+
late Map<String, String> environment;
31+
32+
void printIfNotEmpty(String prefix, String string) {
33+
if (string.isNotEmpty) {
34+
string.split(io.Platform.lineTerminator).forEach((String s) {
35+
print('$prefix:>$s<');
36+
});
37+
}
38+
}
39+
40+
io.ProcessResult run(String executable, List<String> args, {String? workingPath}) {
41+
print('Running "$executable ${args.join(" ")}"${workingPath != null ? ' $workingPath' : ''}');
42+
final io.ProcessResult result = io.Process.runSync(
43+
executable,
44+
args,
45+
environment: environment,
46+
workingDirectory: workingPath ?? testRoot.root.absolute.path,
47+
includeParentEnvironment: false,
48+
);
49+
if (result.exitCode != 0) {
50+
fail(
51+
'Failed running "$executable $args" (exit code = ${result.exitCode}),'
52+
'\nstdout: ${result.stdout}'
53+
'\nstderr: ${result.stderr}',
54+
);
55+
}
56+
printIfNotEmpty('stdout', (result.stdout as String).trim());
57+
printIfNotEmpty('stderr', (result.stderr as String).trim());
58+
return result;
59+
}
60+
61+
setUp(() async {
62+
tmpDir = localFs.systemTempDirectory.createTempSync('content_aware_hash.');
63+
testRoot = _FlutterRootUnderTest.fromPath(tmpDir.childDirectory('flutter').path);
64+
65+
environment = <String, String>{};
66+
67+
if (const LocalPlatform().isWindows) {
68+
// Copy a minimal set of environment variables needed to run the update_engine_version script in PowerShell.
69+
const List<String> powerShellVariables = <String>['SystemRoot', 'Path', 'PATHEXT'];
70+
for (final String key in powerShellVariables) {
71+
final String? value = io.Platform.environment[key];
72+
if (value != null) {
73+
environment[key] = value;
74+
}
75+
}
76+
}
77+
78+
// Make a slim copy of the flutterRoo
79+
flutterRoot.copyTo(testRoot);
80+
});
81+
82+
tearDown(() {
83+
// Git adds a lot of files, we don't want to test for them.
84+
final Directory gitDir = testRoot.root.childDirectory('.git');
85+
if (gitDir.existsSync()) {
86+
gitDir.deleteSync(recursive: true);
87+
}
88+
89+
// Now do cleanup so even if the next step fails, we still deleted tmp.
90+
tmpDir.deleteSync(recursive: true);
91+
});
92+
93+
/// Runs `bin/internal/content_aware_hash.{sh|ps1}` and returns the process result.
94+
///
95+
/// If the exit code is 0, it is considered a success, and files should exist as a side-effect.
96+
/// - On Windows, `powershell` is used (to run `update_engine_version.ps1`);
97+
/// - Otherwise, `update_engine_version.sh` is used.
98+
io.ProcessResult runContentAwareHash() {
99+
final String executable;
100+
final List<String> args;
101+
if (const LocalPlatform().isWindows) {
102+
executable = 'powershell';
103+
args = <String>[testRoot.contentAwareHashPs1.path];
104+
} else {
105+
executable = testRoot.contentAwareHashSh.path;
106+
args = <String>[];
107+
}
108+
return run(executable, args);
109+
}
110+
111+
/// Initializes a blank git repo in [testRoot.root].
112+
void initGitRepoWithBlankInitialCommit({String? workingPath}) {
113+
run('git', <String>['init', '--initial-branch', 'master'], workingPath: workingPath);
114+
// autocrlf is very important for tests to work on windows.
115+
run('git', 'config --local core.autocrlf true'.split(' '), workingPath: workingPath);
116+
run('git', <String>[
117+
'config',
118+
'--local',
119+
'user.email',
120+
121+
], workingPath: workingPath);
122+
run('git', <String>['config', '--local', 'user.name', 'Test User'], workingPath: workingPath);
123+
run('git', <String>['add', '.'], workingPath: workingPath);
124+
run('git', <String>[
125+
'commit',
126+
'--allow-empty',
127+
'-m',
128+
'Initial commit',
129+
], workingPath: workingPath);
130+
}
131+
132+
void writeFileAndCommit(File file, String contents) {
133+
file.writeAsStringSync(contents);
134+
run('git', <String>['add', '--all']);
135+
run('git', <String>['commit', '--all', '-m', 'changed ${file.basename} to $contents']);
136+
}
137+
138+
test('generates a hash', () async {
139+
initGitRepoWithBlankInitialCommit();
140+
expect(runContentAwareHash(), processStdout('eb4bfafe997ec78b3ac8134fbac3eb105ae19155'));
141+
});
142+
143+
group('generates a different hash when', () {
144+
setUp(() {
145+
initGitRepoWithBlankInitialCommit();
146+
});
147+
148+
test('DEPS is changed', () async {
149+
writeFileAndCommit(testRoot.deps, 'deps changed');
150+
expect(runContentAwareHash(), processStdout('38703cae8a58bd0e7e93342bddd20634b069e608'));
151+
});
152+
153+
test('an engine file changes', () async {
154+
writeFileAndCommit(testRoot.engineReadMe, 'engine file changed');
155+
expect(runContentAwareHash(), processStdout('f92b9d9ee03d3530c750235a2fd8559a68d21eac'));
156+
});
157+
158+
test('a new engine file is added', () async {
159+
final List<String> gibberish = ('_abcdefghijklmnopqrstuvqxyz0123456789' * 20).split('')
160+
..shuffle();
161+
final String newFileName = gibberish.take(20).join();
162+
163+
writeFileAndCommit(
164+
testRoot.engineReadMe.parent.childFile(newFileName),
165+
'$newFileName file added',
166+
);
167+
168+
expect(
169+
runContentAwareHash(),
170+
isNot(processStdout('e9d1f7dc1718dac8e8189791a8073e38abdae1cf')),
171+
);
172+
});
173+
174+
test('bin/internal/release-candidate-branch.version is present', () {
175+
writeFileAndCommit(
176+
testRoot.contentAwareHashPs1.parent.childFile('release-candidate-branch.version'),
177+
'sup',
178+
);
179+
expect(runContentAwareHash(), processStdout('f34e6ca2d4dfafc20a5eb23d616df764cbbe937d'));
180+
});
181+
});
182+
183+
test('does not hash non-engine files', () async {
184+
initGitRepoWithBlankInitialCommit();
185+
testRoot.flutterReadMe.writeAsStringSync('codefu was here');
186+
expect(runContentAwareHash(), processStdout('eb4bfafe997ec78b3ac8134fbac3eb105ae19155'));
187+
});
188+
}
189+
190+
/// A FrUT, or "Flutter Root"-Under Test (parallel to a SUT, System Under Test).
191+
///
192+
/// For the intent of this test case, the "Flutter Root" is a directory
193+
/// structure with a minimal set of files.
194+
final class _FlutterRootUnderTest {
195+
/// Creates a root-under test using [path] as the root directory.
196+
///
197+
/// It is assumed the files already exist or will be created if needed.
198+
factory _FlutterRootUnderTest.fromPath(
199+
String path, {
200+
FileSystem fileSystem = const LocalFileSystem(),
201+
}) {
202+
final Directory root = fileSystem.directory(path);
203+
return _FlutterRootUnderTest._(
204+
root,
205+
contentAwareHashPs1: root.childFile(
206+
fileSystem.path.joinAll('bin/internal/content_aware_hash.ps1'.split('/')),
207+
),
208+
contentAwareHashSh: root.childFile(
209+
fileSystem.path.joinAll('bin/internal/content_aware_hash.sh'.split('/')),
210+
),
211+
engineReadMe: root.childFile(fileSystem.path.joinAll('engine/README.md'.split('/'))),
212+
deps: root.childFile(fileSystem.path.join('DEPS')),
213+
flutterReadMe: root.childFile(
214+
fileSystem.path.joinAll('packages/flutter/README.md'.split('/')),
215+
),
216+
);
217+
}
218+
219+
factory _FlutterRootUnderTest.findWithin({
220+
String? path,
221+
FileSystem fileSystem = const LocalFileSystem(),
222+
}) {
223+
path ??= fileSystem.currentDirectory.path;
224+
Directory current = fileSystem.directory(path);
225+
while (!current.childFile('DEPS').existsSync()) {
226+
if (current.path == current.parent.path) {
227+
throw ArgumentError.value(path, 'path', 'Could not resolve flutter root');
228+
}
229+
current = current.parent;
230+
}
231+
return _FlutterRootUnderTest.fromPath(current.path);
232+
}
233+
234+
const _FlutterRootUnderTest._(
235+
this.root, {
236+
required this.deps,
237+
required this.contentAwareHashPs1,
238+
required this.contentAwareHashSh,
239+
required this.engineReadMe,
240+
required this.flutterReadMe,
241+
});
242+
243+
final Directory root;
244+
245+
final File deps;
246+
final File contentAwareHashPs1;
247+
final File contentAwareHashSh;
248+
final File engineReadMe;
249+
final File flutterReadMe;
250+
251+
/// Copies files under test to the [testRoot].
252+
void copyTo(_FlutterRootUnderTest testRoot) {
253+
deps.copySyncRecursive(testRoot.deps.path);
254+
contentAwareHashPs1.copySyncRecursive(testRoot.contentAwareHashPs1.path);
255+
contentAwareHashSh.copySyncRecursive(testRoot.contentAwareHashSh.path);
256+
engineReadMe.copySyncRecursive(testRoot.engineReadMe.path);
257+
flutterReadMe.copySyncRecursive(testRoot.flutterReadMe.path);
258+
}
259+
}
260+
261+
extension on File {
262+
void copySyncRecursive(String newPath) {
263+
fileSystem.directory(fileSystem.path.dirname(newPath)).createSync(recursive: true);
264+
copySync(newPath);
265+
}
266+
}
267+
268+
/// Returns a matcher, that, given [stdout]:
269+
///
270+
/// 1. Process exists with code 0
271+
/// 2. Stdout is a String
272+
/// 3. Stdout contents, after applying [collapseWhitespace], is the same as
273+
/// [stdout], after applying [collapseWhitespace].
274+
Matcher processStdout(String stdout) {
275+
return _ProcessSucceedsAndOutputs(stdout);
276+
}
277+
278+
final class _ProcessSucceedsAndOutputs extends Matcher {
279+
_ProcessSucceedsAndOutputs(String stdout) : _expected = collapseWhitespace(stdout);
280+
281+
final String _expected;
282+
283+
@override
284+
bool matches(Object? item, _) {
285+
if (item is! io.ProcessResult || item.exitCode != 0 || item.stdout is! String) {
286+
return false;
287+
}
288+
final String actual = item.stdout as String;
289+
return collapseWhitespace(actual) == collapseWhitespace(_expected);
290+
}
291+
292+
@override
293+
Description describe(Description description) {
294+
return description.add(
295+
'The process exists normally and stdout (ignoring whitespace): $_expected',
296+
);
297+
}
298+
299+
@override
300+
Description describeMismatch(Object? item, Description mismatch, _, _) {
301+
if (item is! io.ProcessResult) {
302+
return mismatch.add('is not a process result (${item.runtimeType})');
303+
}
304+
if (item.exitCode != 0) {
305+
return mismatch.add('exit code is not zero (${item.exitCode})');
306+
}
307+
if (item.stdout is! String) {
308+
return mismatch.add('stdout is not String (${item.stdout.runtimeType})');
309+
}
310+
return mismatch
311+
.add('is ')
312+
.addDescriptionOf(collapseWhitespace(item.stdout as String))
313+
.add(' with whitespace compressed');
314+
}
315+
}

0 commit comments

Comments
 (0)