Fail snapshot assertions in CI if snapshot is missing

Fixes #1585.

Author: novemberborn

lib/api.js

@@ -207,7 +207,9 @@ class Api extends Emittery {
 
 				const execArgv = await this._computeForkExecArgv();
 				const options = {
-					...apiOptions, // If we're looking for matches, run every single test process in exclusive-only mode
+					...apiOptions,
+					recordNewSnapshots: !isCi,
+					// If we're looking for matches, run every single test process in exclusive-only mode
 					runOnlyExclusive: apiOptions.match.length > 0 || runtimeOptions.runOnlyExclusive === true
 				};
 				if (precompilation) {

lib/assert.js

@@ -616,9 +616,10 @@ class Assertions {
 					values: [formatDescriptorDiff(result.actual, result.expected, {invert: true})]
 				}));
 			} else {
+				// This can only occur in CI environments.
 				fail(new AssertionError({
 					assertion: 'snapshot',
-					message: message || 'No snapshot available, run with --update-snapshots'
+					message: message || 'No snapshot available — new snapshots are not created in CI environments'
 				}));
 			}
 		});

lib/runner.js

@@ -16,6 +16,7 @@ class Runner extends Emittery {
 		this.file = options.file;
 		this.match = options.match || [];
 		this.projectDir = options.projectDir;
+		this.recordNewSnapshots = options.recordNewSnapshots === true;
 		this.runOnlyExclusive = options.runOnlyExclusive === true;
 		this.serial = options.serial === true;
 		this.snapshotDir = options.snapshotDir;
@@ -171,6 +172,7 @@ class Runner extends Emittery {
 				file: this.file,
 				fixedLocation: this.snapshotDir,
 				projectDir: this.projectDir,
+				recordNewSnapshots: this.recordNewSnapshots,
 				updating: this.updateSnapshots
 			});
 			this.emit('dependency', this.snapshots.snapPath);

lib/snapshot-manager.js

@@ -291,6 +291,7 @@ class Manager {
 	constructor(options) {
 		this.appendOnly = options.appendOnly;
 		this.dir = options.dir;
+		this.recordNewSnapshots = options.recordNewSnapshots;
 		this.relFile = options.relFile;
 		this.reportFile = options.reportFile;
 		this.snapFile = options.snapFile;
@@ -309,6 +310,10 @@ class Manager {
 		}
 
 		if (options.index === entries.length) {
+			if (!this.recordNewSnapshots) {
+				return {pass: false};
+			}
+
 			this.record(hash, options);
 			return {pass: true};
 		}
@@ -400,7 +405,7 @@ function resolveSourceFile(file) {
 	return file;
 }
 
-function load({file, fixedLocation, projectDir, updating}) {
+function load({file, fixedLocation, projectDir, recordNewSnapshots, updating}) {
 	const sourceFile = resolveSourceFile(file);
 	const dir = determineSnapshotDir({file: sourceFile, fixedLocation, projectDir});
 	const relFile = path.relative(projectDir, sourceFile);
@@ -424,6 +429,7 @@ function load({file, fixedLocation, projectDir, updating}) {
 	return new Manager({
 		appendOnly,
 		dir,
+		recordNewSnapshots,
 		relFile,
 		reportFile,
 		snapFile,

lib/worker/subprocess.js

@@ -36,6 +36,7 @@ ipc.options.then(options => {
 		file: options.file,
 		match: options.match,
 		projectDir: options.projectDir,
+		recordNewSnapshots: options.recordNewSnapshots,
 		runOnlyExclusive: options.runOnlyExclusive,
 		serial: options.serial,
 		snapshotDir: options.snapshotDir,

test/integration/snapshots.js

@@ -6,6 +6,12 @@ const uniqueTempDir = require('unique-temp-dir');
 const {test} = require('tap');
 const {execCli} = require('../helper/cli');
 
+const overrideCIChecks = {
+	CI: '',
+	TRAVIS: '',
+	CONTINUOUS_INTEGRATION: ''
+};
+
 for (const obj of [
 	{type: 'colocated', rel: '', dir: ''},
 	{type: '__tests__', rel: '__tests__-dir', dir: '__tests__/__snapshots__'},
@@ -24,7 +30,7 @@ for (const obj of [
 
 		const dirname = path.join('fixture/snapshots', obj.rel);
 		// Test should pass, and a snapshot gets written
-		execCli(['--update-snapshots'], {dirname}, error => {
+		execCli(['--update-snapshots', '--verbose'], {dirname, env: overrideCIChecks}, error => {
 			t.ifError(error);
 			t.true(fs.existsSync(snapPath));
 
@@ -50,7 +56,7 @@ test('one', t => {
 })`;
 	fs.writeFileSync(path.join(cwd, 'test.js'), initial);
 
-	const run = () => execa(process.execPath, [cliPath, '--verbose', '--no-color'], {cwd, env: {CI: '1'}, reject: false});
+	const run = () => execa(process.execPath, [cliPath, '--verbose', '--no-color'], {cwd, env: overrideCIChecks, reject: false});
 	return run().then(result => {
 		t.match(result.stdout, /1 test passed/);
 
@@ -161,7 +167,7 @@ test('snapshots infer their location and name from sourcemaps', t => {
 		t.true(fs.existsSync(relFilePath));
 	};
 
-	execCli([], {dirname: relativeFixtureDir}, (error, stdout) => {
+	execCli(['--verbose'], {dirname: relativeFixtureDir, env: overrideCIChecks}, (error, stdout) => {
 		t.ifError(error);
 		snapFixtureFilePaths.forEach(x => verifySnapFixtureFiles(x));
 		t.match(stdout, /6 tests passed/);
@@ -202,7 +208,7 @@ test('snapshots resolved location from "snapshotDir" in AVA config', t => {
 		t.true(fs.existsSync(relFilePath));
 	};
 
-	execCli([], {dirname: relativeFixtureDir}, (error, stdout) => {
+	execCli(['--verbose'], {dirname: relativeFixtureDir, env: overrideCIChecks}, (error, stdout) => {
 		t.ifError(error);
 		snapFixtureFilePaths.forEach(x => verifySnapFixtureFiles(x));
 		t.match(stdout, /6 tests passed/);
@@ -231,7 +237,7 @@ test('snapshots are indentical on different platforms', t => {
 	[reportPath, snapPath].forEach(fp => removeFile(fp));
 
 	// Test should pass, and a snapshot gets written
-	execCli(['--update-snapshots'], {dirname: fixtureDir}, error => {
+	execCli(['--update-snapshots', '--verbose'], {dirname: fixtureDir, env: overrideCIChecks}, error => {
 		t.ifError(error);
 		t.true(fs.existsSync(reportPath));
 		t.true(fs.existsSync(snapPath));
@@ -246,3 +252,30 @@ test('snapshots are indentical on different platforms', t => {
 		t.end();
 	});
 });
+
+test('in CI, new snapshots are not recorded', t => {
+	const fixtureDir = path.join(__dirname, '..', 'fixture', 'snapshots', 'test-content');
+	const reportPath = path.join(fixtureDir, 'tests', 'snapshots', 'test.js.md');
+	const snapPath = path.join(fixtureDir, 'tests', 'snapshots', 'test.js.snap');
+
+	const removeFile = filePath => {
+		try {
+			fs.unlinkSync(filePath);
+		} catch (error) {
+			if (error.code !== 'ENOENT') {
+				throw error;
+			}
+		}
+	};
+
+	// Clear current snapshots
+	[reportPath, snapPath].forEach(fp => removeFile(fp));
+
+	// Test should fail, no snapshot gets written
+	execCli([], {dirname: fixtureDir}, (_, stdout) => {
+		t.match(stdout, 'No snapshot available — new snapshots are not created in CI environments');
+		t.false(fs.existsSync(reportPath));
+		t.false(fs.existsSync(snapPath));
+		t.end();
+	});
+});